| Trees | Indices | Help |
|
|---|
|
|
1 # This file is part of Buildbot. Buildbot is free software: you can
2 # redistribute it and/or modify it under the terms of the GNU General Public
3 # License as published by the Free Software Foundation, version 2.
4 #
5 # This program is distributed in the hope that it will be useful, but WITHOUT
6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
8 # details.
9 #
10 # You should have received a copy of the GNU General Public License along with
11 # this program; if not, write to the Free Software Foundation, Inc., 51
12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
13 #
14 # Portions Copyright Buildbot Team Members
15 # Original Copyright (c) 2010 The Chromium Authors.
16
17 """Simple JSON exporter."""
18
19 import datetime
20 import os
21 import re
22
23 from twisted.internet import defer
24 from twisted.web import html, resource, server
25
26 from buildbot.status.web.base import HtmlResource
27 from buildbot.util import json
28
29
30 _IS_INT = re.compile('^[-+]?\d+$')
31
32
33 FLAGS = """\
34 - as_text
35 - By default, application/json is used. Setting as_text=1 change the type
36 to text/plain and implicitly sets compact=0 and filter=1. Mainly useful to
37 look at the result in a web browser.
38 - compact
39 - By default, the json data is compact and defaults to 1. For easier to read
40 indented output, set compact=0.
41 - select
42 - By default, most children data is listed. You can do a random selection
43 of data by using select=<sub-url> multiple times to coagulate data.
44 "select=" includes the actual url otherwise it is skipped.
45 - numbuilds
46 - By default, only in memory cached builds are listed. You can as for more data
47 by using numbuilds=<number>.
48 - filter
49 - Filters out null, false, and empty string, list and dict. This reduce the
50 amount of useless data sent.
51 - callback
52 - Enable uses of JSONP as described in
53 http://en.wikipedia.org/wiki/JSONP. Note that
54 Access-Control-Allow-Origin:* is set in the HTTP response header so you
55 can use this in compatible browsers.
56 """
57
58 EXAMPLES = """\
59 - /json
60 - Root node, that *doesn't* mean all the data. Many things (like logs) must
61 be explicitly queried for performance reasons.
62 - /json/builders/
63 - All builders.
64 - /json/builders/<A_BUILDER>
65 - A specific builder as compact text.
66 - /json/builders/<A_BUILDER>/builds
67 - All *cached* builds.
68 - /json/builders/<A_BUILDER>/builds/_all
69 - All builds. Warning, reads all previous build data.
70 - /json/builders/<A_BUILDER>/builds/<A_BUILD>
71 - Where <A_BUILD> is either positive, a build number, or negative, a past
72 build.
73 - /json/builders/<A_BUILDER>/builds/-1/source_stamp/changes
74 - Build changes
75 - /json/builders/<A_BUILDER>/builds?select=-1&select=-2
76 - Two last builds on '<A_BUILDER>' builder.
77 - /json/builders/<A_BUILDER>/builds?select=-1/source_stamp/changes&select=-2/source_stamp/changes
78 - Changes of the two last builds on '<A_BUILDER>' builder.
79 - /json/builders/<A_BUILDER>/slaves
80 - Slaves associated to this builder.
81 - /json/builders/<A_BUILDER>?select=&select=slaves
82 - Builder information plus details information about its slaves. Neat eh?
83 - /json/slaves/<A_SLAVE>
84 - A specific slave.
85 - /json?select=slaves/<A_SLAVE>/&select=project&select=builders/<A_BUILDER>/builds/<A_BUILD>
86 - A selection of random unrelated stuff as an random example. :)
87 """
92
95 value = RequestArg(request, arg, default)
96 if value in (False, True):
97 return value
98 value = value.lower()
99 if value in ('1', 'true'):
100 return True
101 if value in ('0', 'false'):
102 return False
103 # Ignore value.
104 return default
105
108 """Returns a copy with None, False, "", [], () and {} removed.
109 Warning: converts tuple to list."""
110 if isinstance(data, (list, tuple)):
111 # Recurse in every items and filter them out.
112 items = map(FilterOut, data)
113 if not filter(lambda x: not x in ('', False, None, [], {}, ()), items):
114 return None
115 return items
116 elif isinstance(data, dict):
117 return dict(filter(lambda x: not x[1] in ('', False, None, [], {}, ()),
118 [(k, FilterOut(v)) for (k, v) in data.iteritems()]))
119 else:
120 return data
121
124 """Base class for json data."""
125
126 contentType = "application/json"
127 cache_seconds = 60
128 help = None
129 pageTitle = None
130 level = 0
131
133 """Adds transparent lazy-child initialization."""
134 resource.Resource.__init__(self)
135 # buildbot.status.builder.Status
136 self.status = status
137
139 """Adds transparent support for url ending with /"""
140 if path == "" and len(request.postpath) == 0:
141 return self
142 if path == 'help' and self.help:
143 pageTitle = ''
144 if self.pageTitle:
145 pageTitle = self.pageTitle + ' help'
146 return HelpResource(self.help,
147 pageTitle=pageTitle,
148 parent_node=self)
149 # Equivalent to resource.Resource.getChildWithDefault()
150 if self.children.has_key(path):
151 return self.children[path]
152 return self.getChild(path, request)
153
155 """Adds the resource's level for help links generation."""
156
157 def RecurseFix(res, level):
158 res.level = level + 1
159 for c in res.children.itervalues():
160 RecurseFix(c, res.level)
161
162 RecurseFix(res, self.level)
163 resource.Resource.putChild(self, name, res)
164
166 """Renders a HTTP GET at the http request level."""
167 d = defer.maybeDeferred(lambda : self.content(request))
168 def handle(data):
169 if isinstance(data, unicode):
170 data = data.encode("utf-8")
171 request.setHeader("Access-Control-Allow-Origin", "*")
172 if RequestArgToBool(request, 'as_text', False):
173 request.setHeader("content-type", 'text/plain')
174 else:
175 request.setHeader("content-type", self.contentType)
176 request.setHeader("content-disposition",
177 "attachment; filename=\"%s.json\"" % request.path)
178 # Make sure we get fresh pages.
179 if self.cache_seconds:
180 now = datetime.datetime.utcnow()
181 expires = now + datetime.timedelta(seconds=self.cache_seconds)
182 request.setHeader("Expires",
183 expires.strftime("%a, %d %b %Y %H:%M:%S GMT"))
184 request.setHeader("Pragma", "no-cache")
185 return data
186 d.addCallback(handle)
187 def ok(data):
188 request.write(data)
189 request.finish()
190 def fail(f):
191 request.processingFailed(f)
192 return None # processingFailed will log this for us
193 d.addCallbacks(ok, fail)
194 return server.NOT_DONE_YET
195
196 @defer.inlineCallbacks
198 """Renders the json dictionaries."""
199 # Supported flags.
200 select = request.args.get('select')
201 as_text = RequestArgToBool(request, 'as_text', False)
202 filter_out = RequestArgToBool(request, 'filter', as_text)
203 compact = RequestArgToBool(request, 'compact', not as_text)
204 callback = request.args.get('callback')
205
206 # Implement filtering at global level and every child.
207 if select is not None:
208 del request.args['select']
209 # Do not render self.asDict()!
210 data = {}
211 # Remove superfluous /
212 select = [s.strip('/') for s in select]
213 select.sort(cmp=lambda x,y: cmp(x.count('/'), y.count('/')),
214 reverse=True)
215 for item in select:
216 # Start back at root.
217 node = data
218 # Implementation similar to twisted.web.resource.getChildForRequest
219 # but with a hacked up request.
220 child = self
221 prepath = request.prepath[:]
222 postpath = request.postpath[:]
223 request.postpath = filter(None, item.split('/'))
224 while request.postpath and not child.isLeaf:
225 pathElement = request.postpath.pop(0)
226 node[pathElement] = {}
227 node = node[pathElement]
228 request.prepath.append(pathElement)
229 child = child.getChildWithDefault(pathElement, request)
230
231 # some asDict methods return a Deferred, so handle that
232 # properly
233 if hasattr(child, 'asDict'):
234 child_dict = yield defer.maybeDeferred(lambda :
235 child.asDict(request))
236 else:
237 child_dict = {
238 'error' : 'Not available',
239 }
240 node.update(child_dict)
241
242 request.prepath = prepath
243 request.postpath = postpath
244 else:
245 data = yield defer.maybeDeferred(lambda : self.asDict(request))
246
247 if filter_out:
248 data = FilterOut(data)
249 if compact:
250 data = json.dumps(data, sort_keys=True, separators=(',',':'))
251 else:
252 data = json.dumps(data, sort_keys=True, indent=2)
253 if callback:
254 # Only accept things that look like identifiers for now
255 callback = callback[0]
256 if re.match(r'^[a-zA-Z$][a-zA-Z$0-9.]*$', callback):
257 data = '%s(%s);' % (callback, data)
258 defer.returnValue(data)
259
260 @defer.inlineCallbacks
262 """Generates the json dictionary.
263
264 By default, renders every childs."""
265 if self.children:
266 data = {}
267 for name in self.children:
268 child = self.getChildWithDefault(name, request)
269 if isinstance(child, JsonResource):
270 data[name] = yield defer.maybeDeferred(lambda :
271 child.asDict(request))
272 # else silently pass over non-json resources.
273 defer.returnValue(data)
274 else:
275 raise NotImplementedError()
276
279 """Convert a string in a wiki-style format into HTML."""
280 indent = 0
281 in_item = False
282 output = []
283 for line in text.splitlines(False):
284 match = re.match(r'^( +)\- (.*)$', line)
285 if match:
286 if indent < len(match.group(1)):
287 output.append('<ul>')
288 indent = len(match.group(1))
289 elif indent > len(match.group(1)):
290 while indent > len(match.group(1)):
291 output.append('</ul>')
292 indent -= 2
293 if in_item:
294 # Close previous item
295 output.append('</li>')
296 output.append('<li>')
297 in_item = True
298 line = match.group(2)
299 elif indent:
300 if line.startswith((' ' * indent) + ' '):
301 # List continuation
302 line = line.strip()
303 else:
304 # List is done
305 if in_item:
306 output.append('</li>')
307 in_item = False
308 while indent > 0:
309 output.append('</ul>')
310 indent -= 2
311
312 if line.startswith('/'):
313 if not '?' in line:
314 line_full = line + '?as_text=1'
315 else:
316 line_full = line + '&as_text=1'
317 output.append('<a href="' + html.escape(line_full) + '">' +
318 html.escape(line) + '</a>')
319 else:
320 output.append(html.escape(line).replace(' ', ' '))
321 if not in_item:
322 output.append('<br>')
323
324 if in_item:
325 output.append('</li>')
326 while indent > 0:
327 output.append('</ul>')
328 indent -= 2
329 return '\n'.join(output)
330
334 HtmlResource.__init__(self)
335 self.text = text
336 self.pageTitle = pageTitle
337 self.parent_level = parent_node.level
338 self.parent_children = parent_node.children.keys()
339
341 cxt['level'] = self.parent_level
342 cxt['text'] = ToHtml(self.text)
343 cxt['children'] = [ n for n in self.parent_children if n != 'help' ]
344 cxt['flags'] = ToHtml(FLAGS)
345 cxt['examples'] = ToHtml(EXAMPLES).replace(
346 'href="/json',
347 'href="../%sjson' % (self.parent_level * '../'))
348
349 template = request.site.buildbot_service.templates.get_template("jsonhelp.html")
350 return template.render(**cxt)
351
353 help = """Describe pending builds for a builder.
354 """
355 pageTitle = 'Builder'
356
360
362 # buildbot.status.builder.BuilderStatus
363 d = self.builder_status.getPendingBuildRequestStatuses()
364 def to_dict(statuses):
365 return defer.gatherResults(
366 [ b.asDict_async() for b in statuses ])
367 d.addCallback(to_dict)
368 return d
369
372 help = """Describe a single builder.
373 """
374 pageTitle = 'Builder'
375
377 JsonResource.__init__(self, status)
378 self.builder_status = builder_status
379 self.putChild('builds', BuildsJsonResource(status, builder_status))
380 self.putChild('slaves', BuilderSlavesJsonResources(status,
381 builder_status))
382 self.putChild(
383 'pendingBuilds',
384 BuilderPendingBuildsJsonResource(status, builder_status))
385
389
392 help = """List of all the builders defined on a master.
393 """
394 pageTitle = 'Builders'
395
397 JsonResource.__init__(self, status)
398 for builder_name in self.status.getBuilderNames():
399 self.putChild(builder_name,
400 BuilderJsonResource(status,
401 status.getBuilder(builder_name)))
402
405 help = """Describe the slaves attached to a single builder.
406 """
407 pageTitle = 'BuilderSlaves'
408
410 JsonResource.__init__(self, status)
411 self.builder_status = builder_status
412 for slave_name in self.builder_status.slavenames:
413 self.putChild(slave_name,
414 SlaveJsonResource(status,
415 self.status.getSlave(slave_name)))
416
419 help = """Describe a single build.
420 """
421 pageTitle = 'Build'
422
424 JsonResource.__init__(self, status)
425 self.build_status = build_status
426 # TODO: support multiple sourcestamps
427 sourcestamp = build_status.getSourceStamps()[0]
428 self.putChild('source_stamp',
429 SourceStampJsonResource(status, sourcestamp))
430 self.putChild('steps', BuildStepsJsonResource(status, build_status))
431
434
437 help = """All the builds that were run on a builder.
438 """
439 pageTitle = 'AllBuilds'
440
444
446 # Dynamic childs.
447 if isinstance(path, int) or _IS_INT.match(path):
448 build_status = self.builder_status.getBuild(int(path))
449 if build_status:
450 return BuildJsonResource(self.status, build_status)
451 return JsonResource.getChild(self, path, request)
452
454 results = {}
455 # If max > buildCacheSize, it'll trash the cache...
456 cache_size = self.builder_status.master.config.caches['Builds']
457 max = int(RequestArg(request, 'max', cache_size))
458 for i in range(0, max):
459 child = self.getChildWithDefault(-i, request)
460 if not isinstance(child, BuildJsonResource):
461 continue
462 results[child.build_status.getNumber()] = child.asDict(request)
463 return results
464
467 help = """Builds that were run on a builder.
468 """
469 pageTitle = 'Builds'
470
472 AllBuildsJsonResource.__init__(self, status, builder_status)
473 self.putChild('_all', AllBuildsJsonResource(status, builder_status))
474
476 # Transparently redirects to _all if path is not ''.
477 return self.children['_all'].getChildWithDefault(path, request)
478
480 # This would load all the pickles and is way too heavy, especially that
481 # it would trash the cache:
482 # self.children['builds'].asDict(request)
483 # TODO(maruel) This list should also need to be cached but how?
484 builds = dict([
485 (int(file), None)
486 for file in os.listdir(self.builder_status.basedir)
487 if _IS_INT.match(file)
488 ])
489 return builds
490
493 help = """A single build step.
494 """
495 pageTitle = 'BuildStep'
496
498 # buildbot.status.buildstep.BuildStepStatus
499 JsonResource.__init__(self, status)
500 self.build_step_status = build_step_status
501 # TODO self.putChild('logs', LogsJsonResource())
502
504 return self.build_step_status.asDict()
505
508 help = """A list of build steps that occurred during a build.
509 """
510 pageTitle = 'BuildSteps'
511
515 # The build steps are constantly changing until the build is done so
516 # keep a reference to build_status instead
517
519 # Dynamic childs.
520 build_step_status = None
521 if isinstance(path, int) or _IS_INT.match(path):
522 build_step_status = self.build_status.getSteps()[int(path)]
523 else:
524 steps_dict = dict([(step.getName(), step)
525 for step in self.build_status.getSteps()])
526 build_step_status = steps_dict.get(path)
527 if build_step_status:
528 # Create it on-demand.
529 child = BuildStepJsonResource(self.status, build_step_status)
530 # Cache it.
531 index = self.build_status.getSteps().index(build_step_status)
532 self.putChild(str(index), child)
533 self.putChild(build_step_status.getName(), child)
534 return child
535 return JsonResource.getChild(self, path, request)
536
545
548 help = """Describe a single change that originates from a change source.
549 """
550 pageTitle = 'Change'
551
553 # buildbot.changes.changes.Change
554 JsonResource.__init__(self, status)
555 self.change = change
556
558 return self.change.asDict()
559
562 help = """List of changes.
563 """
564 pageTitle = 'Changes'
565
567 JsonResource.__init__(self, status)
568 for c in changes:
569 # c.number can be None or clash another change if the change was
570 # generated inside buildbot or if using multiple pollers.
571 if c.number is not None and str(c.number) not in self.children:
572 self.putChild(str(c.number), ChangeJsonResource(status, c))
573 else:
574 # Temporary hack since it creates information exposure.
575 self.putChild(str(id(c)), ChangeJsonResource(status, c))
576
578 """Don't throw an exception when there is no child."""
579 if not self.children:
580 return {}
581 return JsonResource.asDict(self, request)
582
585 help = """Describe a change source.
586 """
587 pageTitle = 'ChangeSources'
588
590 result = {}
591 n = 0
592 for c in self.status.getChangeSources():
593 # buildbot.changes.changes.ChangeMaster
594 change = {}
595 change['description'] = c.describe()
596 result[n] = change
597 n += 1
598 return result
599
608
611 help = """Describe a slave.
612 """
613 pageTitle = 'Slave'
614
616 JsonResource.__init__(self, status)
617 self.slave_status = slave_status
618 self.name = self.slave_status.getName()
619 self.builders = None
620
622 if self.builders is None:
623 # Figure out all the builders to which it's attached
624 self.builders = []
625 for builderName in self.status.getBuilderNames():
626 if self.name in self.status.getBuilder(builderName).slavenames:
627 self.builders.append(builderName)
628 return self.builders
629
631 results = self.slave_status.asDict()
632 # Enhance it by adding more informations.
633 results['builders'] = {}
634 for builderName in self.getBuilders():
635 builds = []
636 builder_status = self.status.getBuilder(builderName)
637 cache_size = builder_status.master.config.caches['Builds']
638 numbuilds = int(request.args.get('numbuilds', [cache_size - 1])[0])
639 for i in range(1, numbuilds):
640 build_status = builder_status.getBuild(-i)
641 if not build_status or not build_status.isFinished():
642 # If not finished, it will appear in runningBuilds.
643 break
644 if build_status.getSlavename() == self.name:
645 builds.append(build_status.getNumber())
646 results['builders'][builderName] = builds
647 return results
648
651 help = """List the registered slaves.
652 """
653 pageTitle = 'Slaves'
654
656 JsonResource.__init__(self, status)
657 for slave_name in status.getSlaveNames():
658 self.putChild(slave_name,
659 SlaveJsonResource(status,
660 status.getSlave(slave_name)))
661
664 help = """Describe the sources for a SourceStamp.
665 """
666 pageTitle = 'SourceStamp'
667
669 # buildbot.sourcestamp.SourceStamp
670 JsonResource.__init__(self, status)
671 self.source_stamp = source_stamp
672 self.putChild('changes',
673 ChangesJsonResource(status, source_stamp.changes))
674 # TODO(maruel): Should redirect to the patch's url instead.
675 #if source_stamp.patch:
676 # self.putChild('patch', StaticHTML(source_stamp.path))
677
679 return self.source_stamp.asDict()
680
693
697 """Retrieves all json data."""
698 help = """JSON status
699
700 Root page to give a fair amount of information in the current buildbot master
701 status. You may want to use a child instead to reduce the load on the server.
702
703 For help on any sub directory, use url /child/help
704 """
705 pageTitle = 'Buildbot JSON'
706
708 JsonResource.__init__(self, status)
709 self.level = 1
710 self.putChild('builders', BuildersJsonResource(status))
711 self.putChild('change_sources', ChangeSourcesJsonResource(status))
712 self.putChild('project', ProjectJsonResource(status))
713 self.putChild('slaves', SlavesJsonResource(status))
714 self.putChild('metrics', MetricsJsonResource(status))
715 # This needs to be called before the first HelpResource().body call.
716 self.hackExamples()
717
719 result = JsonResource.content(self, request)
720 # This is done to hook the downloaded filename.
721 request.path = 'buildbot'
722 return result
723
725 global EXAMPLES
726 # Find the first builder with a previous build or select the last one.
727 builder = None
728 for b in self.status.getBuilderNames():
729 builder = self.status.getBuilder(b)
730 if builder.getBuild(-1):
731 break
732 if not builder:
733 return
734 EXAMPLES = EXAMPLES.replace('<A_BUILDER>', builder.getName())
735 build = builder.getBuild(-1)
736 if build:
737 EXAMPLES = EXAMPLES.replace('<A_BUILD>', str(build.getNumber()))
738 if builder.slavenames:
739 EXAMPLES = EXAMPLES.replace('<A_SLAVE>', builder.slavenames[0])
740
741 # vim: set ts=4 sts=4 sw=4 et:
742
| Trees | Indices | Help |
|
|---|
| Generated by Epydoc 3.0.1 on Wed Nov 21 16:22:51 2012 | http://epydoc.sourceforge.net |