| Trees | Indices | Help |
|
|---|
|
|
1 # This file is part of Buildbot. Buildbot is free software: you can
2 # redistribute it and/or modify it under the terms of the GNU General Public
3 # License as published by the Free Software Foundation, version 2.
4 #
5 # This program is distributed in the hope that it will be useful, but WITHOUT
6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
8 # details.
9 #
10 # You should have received a copy of the GNU General Public License along with
11 # this program; if not, write to the Free Software Foundation, Inc., 51
12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
13 #
14 # Portions Copyright Buildbot Team Members
15 # Original Copyright (c) 2010 The Chromium Authors.
16
17 """Simple JSON exporter."""
18
19 import datetime
20 import os
21 import re
22
23 from twisted.internet import defer
24 from twisted.web import html, resource, server
25
26 from buildbot.status.web.base import HtmlResource
27 from buildbot.util import json
28
29
30 _IS_INT = re.compile('^[-+]?\d+$')
31
32
33 FLAGS = """\
34 - as_text
35 - By default, application/json is used. Setting as_text=1 change the type
36 to text/plain and implicitly sets compact=0 and filter=1. Mainly useful to
37 look at the result in a web browser.
38 - compact
39 - By default, the json data is compact and defaults to 1. For easier to read
40 indented output, set compact=0.
41 - select
42 - By default, most children data is listed. You can do a random selection
43 of data by using select=<sub-url> multiple times to coagulate data.
44 "select=" includes the actual url otherwise it is skipped.
45 - numbuilds
46 - By default, only in memory cached builds are listed. You can as for more data
47 by using numbuilds=<number>.
48 - filter
49 - Filters out null, false, and empty string, list and dict. This reduce the
50 amount of useless data sent.
51 - callback
52 - Enable uses of JSONP as described in
53 http://en.wikipedia.org/wiki/JSONP. Note that
54 Access-Control-Allow-Origin:* is set in the HTTP response header so you
55 can use this in compatible browsers.
56 """
57
58 EXAMPLES = """\
59 - /json
60 - Root node, that *doesn't* mean all the data. Many things (like logs) must
61 be explicitly queried for performance reasons.
62 - /json/builders/
63 - All builders.
64 - /json/builders/<A_BUILDER>
65 - A specific builder as compact text.
66 - /json/builders/<A_BUILDER>/builds
67 - All *cached* builds.
68 - /json/builders/<A_BUILDER>/builds/_all
69 - All builds. Warning, reads all previous build data.
70 - /json/builders/<A_BUILDER>/builds/<A_BUILD>
71 - Where <A_BUILD> is either positive, a build number, or negative, a past
72 build.
73 - /json/builders/<A_BUILDER>/builds/-1/source_stamp/changes
74 - Build changes
75 - /json/builders/<A_BUILDER>/builds?select=-1&select=-2
76 - Two last builds on '<A_BUILDER>' builder.
77 - /json/builders/<A_BUILDER>/builds?select=-1/source_stamp/changes&select=-2/source_stamp/changes
78 - Changes of the two last builds on '<A_BUILDER>' builder.
79 - /json/builders/<A_BUILDER>/slaves
80 - Slaves associated to this builder.
81 - /json/builders/<A_BUILDER>?select=&select=slaves
82 - Builder information plus details information about its slaves. Neat eh?
83 - /json/slaves/<A_SLAVE>
84 - A specific slave.
85 - /json?select=slaves/<A_SLAVE>/&select=project&select=builders/<A_BUILDER>/builds/<A_BUILD>
86 - A selection of random unrelated stuff as an random example. :)
87 """
92
95 value = RequestArg(request, arg, default)
96 if value in (False, True):
97 return value
98 value = value.lower()
99 if value in ('1', 'true'):
100 return True
101 if value in ('0', 'false'):
102 return False
103 # Ignore value.
104 return default
105
108 """Returns a copy with None, False, "", [], () and {} removed.
109 Warning: converts tuple to list."""
110 if isinstance(data, (list, tuple)):
111 # Recurse in every items and filter them out.
112 items = map(FilterOut, data)
113 if not filter(lambda x: not x in ('', False, None, [], {}, ()), items):
114 return None
115 return items
116 elif isinstance(data, dict):
117 return dict(filter(lambda x: not x[1] in ('', False, None, [], {}, ()),
118 [(k, FilterOut(v)) for (k, v) in data.iteritems()]))
119 else:
120 return data
121
124 """Base class for json data."""
125
126 contentType = "application/json"
127 cache_seconds = 60
128 help = None
129 pageTitle = None
130 level = 0
131
133 """Adds transparent lazy-child initialization."""
134 resource.Resource.__init__(self)
135 # buildbot.status.builder.Status
136 self.status = status
137 if self.help:
138 pageTitle = ''
139 if self.pageTitle:
140 pageTitle = self.pageTitle + ' help'
141 self.putChild('help',
142 HelpResource(self.help, pageTitle=pageTitle, parent_node=self))
143
145 """Adds transparent support for url ending with /"""
146 if path == "" and len(request.postpath) == 0:
147 return self
148 # Equivalent to resource.Resource.getChildWithDefault()
149 if self.children.has_key(path):
150 return self.children[path]
151 return self.getChild(path, request)
152
154 """Adds the resource's level for help links generation."""
155
156 def RecurseFix(res, level):
157 res.level = level + 1
158 for c in res.children.itervalues():
159 RecurseFix(c, res.level)
160
161 RecurseFix(res, self.level)
162 resource.Resource.putChild(self, name, res)
163
165 """Renders a HTTP GET at the http request level."""
166 d = defer.maybeDeferred(lambda : self.content(request))
167 def handle(data):
168 if isinstance(data, unicode):
169 data = data.encode("utf-8")
170 request.setHeader("Access-Control-Allow-Origin", "*")
171 if RequestArgToBool(request, 'as_text', False):
172 request.setHeader("content-type", 'text/plain')
173 else:
174 request.setHeader("content-type", self.contentType)
175 request.setHeader("content-disposition",
176 "attachment; filename=\"%s.json\"" % request.path)
177 # Make sure we get fresh pages.
178 if self.cache_seconds:
179 now = datetime.datetime.utcnow()
180 expires = now + datetime.timedelta(seconds=self.cache_seconds)
181 request.setHeader("Expires",
182 expires.strftime("%a, %d %b %Y %H:%M:%S GMT"))
183 request.setHeader("Pragma", "no-cache")
184 return data
185 d.addCallback(handle)
186 def ok(data):
187 request.write(data)
188 request.finish()
189 def fail(f):
190 request.processingFailed(f)
191 return None # processingFailed will log this for us
192 d.addCallbacks(ok, fail)
193 return server.NOT_DONE_YET
194
195 @defer.deferredGenerator
197 """Renders the json dictionaries."""
198 # Supported flags.
199 select = request.args.get('select')
200 as_text = RequestArgToBool(request, 'as_text', False)
201 filter_out = RequestArgToBool(request, 'filter', as_text)
202 compact = RequestArgToBool(request, 'compact', not as_text)
203 callback = request.args.get('callback')
204
205 # Implement filtering at global level and every child.
206 if select is not None:
207 del request.args['select']
208 # Do not render self.asDict()!
209 data = {}
210 # Remove superfluous /
211 select = [s.strip('/') for s in select]
212 select.sort(cmp=lambda x,y: cmp(x.count('/'), y.count('/')),
213 reverse=True)
214 for item in select:
215 # Start back at root.
216 node = data
217 # Implementation similar to twisted.web.resource.getChildForRequest
218 # but with a hacked up request.
219 child = self
220 prepath = request.prepath[:]
221 postpath = request.postpath[:]
222 request.postpath = filter(None, item.split('/'))
223 while request.postpath and not child.isLeaf:
224 pathElement = request.postpath.pop(0)
225 node[pathElement] = {}
226 node = node[pathElement]
227 request.prepath.append(pathElement)
228 child = child.getChildWithDefault(pathElement, request)
229
230 # some asDict methods return a Deferred, so handle that
231 # properly
232 if hasattr(child, 'asDict'):
233 wfd = defer.waitForDeferred(
234 defer.maybeDeferred(lambda :
235 child.asDict(request)))
236 yield wfd
237 child_dict = wfd.getResult()
238 else:
239 child_dict = {
240 'error' : 'Not available',
241 }
242 node.update(child_dict)
243
244 request.prepath = prepath
245 request.postpath = postpath
246 else:
247 wfd = defer.waitForDeferred(
248 defer.maybeDeferred(lambda :
249 self.asDict(request)))
250 yield wfd
251 data = wfd.getResult()
252
253 if filter_out:
254 data = FilterOut(data)
255 if compact:
256 data = json.dumps(data, sort_keys=True, separators=(',',':'))
257 else:
258 data = json.dumps(data, sort_keys=True, indent=2)
259 if callback:
260 # Only accept things that look like identifiers for now
261 callback = callback[0]
262 if re.match(r'^[a-zA-Z$][a-zA-Z$0-9.]*$', callback):
263 data = '%s(%s);' % (callback, data)
264 yield data
265
266 @defer.deferredGenerator
268 """Generates the json dictionary.
269
270 By default, renders every childs."""
271 if self.children:
272 data = {}
273 for name in self.children:
274 child = self.getChildWithDefault(name, request)
275 if isinstance(child, JsonResource):
276 wfd = defer.waitForDeferred(
277 defer.maybeDeferred(lambda :
278 child.asDict(request)))
279 yield wfd
280 data[name] = wfd.getResult()
281 # else silently pass over non-json resources.
282 yield data
283 else:
284 raise NotImplementedError()
285
288 """Convert a string in a wiki-style format into HTML."""
289 indent = 0
290 in_item = False
291 output = []
292 for line in text.splitlines(False):
293 match = re.match(r'^( +)\- (.*)$', line)
294 if match:
295 if indent < len(match.group(1)):
296 output.append('<ul>')
297 indent = len(match.group(1))
298 elif indent > len(match.group(1)):
299 while indent > len(match.group(1)):
300 output.append('</ul>')
301 indent -= 2
302 if in_item:
303 # Close previous item
304 output.append('</li>')
305 output.append('<li>')
306 in_item = True
307 line = match.group(2)
308 elif indent:
309 if line.startswith((' ' * indent) + ' '):
310 # List continuation
311 line = line.strip()
312 else:
313 # List is done
314 if in_item:
315 output.append('</li>')
316 in_item = False
317 while indent > 0:
318 output.append('</ul>')
319 indent -= 2
320
321 if line.startswith('/'):
322 if not '?' in line:
323 line_full = line + '?as_text=1'
324 else:
325 line_full = line + '&as_text=1'
326 output.append('<a href="' + html.escape(line_full) + '">' +
327 html.escape(line) + '</a>')
328 else:
329 output.append(html.escape(line).replace(' ', ' '))
330 if not in_item:
331 output.append('<br>')
332
333 if in_item:
334 output.append('</li>')
335 while indent > 0:
336 output.append('</ul>')
337 indent -= 2
338 return '\n'.join(output)
339
343 HtmlResource.__init__(self)
344 self.text = text
345 self.pageTitle = pageTitle
346 self.parent_node = parent_node
347
349 cxt['level'] = self.parent_node.level
350 cxt['text'] = ToHtml(self.text)
351 cxt['children'] = [ n for n in self.parent_node.children.keys() if n != 'help' ]
352 cxt['flags'] = ToHtml(FLAGS)
353 cxt['examples'] = ToHtml(EXAMPLES).replace(
354 'href="/json',
355 'href="%sjson' % (self.level * '../'))
356
357 template = request.site.buildbot_service.templates.get_template("jsonhelp.html")
358 return template.render(**cxt)
359
361 help = """Describe pending builds for a builder.
362 """
363 pageTitle = 'Builder'
364
368
370 # buildbot.status.builder.BuilderStatus
371 d = self.builder_status.getPendingBuildRequestStatuses()
372 def to_dict(statuses):
373 return defer.gatherResults(
374 [ b.asDict_async() for b in statuses ])
375 d.addCallback(to_dict)
376 return d
377
380 help = """Describe a single builder.
381 """
382 pageTitle = 'Builder'
383
385 JsonResource.__init__(self, status)
386 self.builder_status = builder_status
387 self.putChild('builds', BuildsJsonResource(status, builder_status))
388 self.putChild('slaves', BuilderSlavesJsonResources(status,
389 builder_status))
390 self.putChild(
391 'pendingBuilds',
392 BuilderPendingBuildsJsonResource(status, builder_status))
393
397
400 help = """List of all the builders defined on a master.
401 """
402 pageTitle = 'Builders'
403
405 JsonResource.__init__(self, status)
406 for builder_name in self.status.getBuilderNames():
407 self.putChild(builder_name,
408 BuilderJsonResource(status,
409 status.getBuilder(builder_name)))
410
413 help = """Describe the slaves attached to a single builder.
414 """
415 pageTitle = 'BuilderSlaves'
416
418 JsonResource.__init__(self, status)
419 self.builder_status = builder_status
420 for slave_name in self.builder_status.slavenames:
421 self.putChild(slave_name,
422 SlaveJsonResource(status,
423 self.status.getSlave(slave_name)))
424
427 help = """Describe a single build.
428 """
429 pageTitle = 'Build'
430
432 JsonResource.__init__(self, status)
433 self.build_status = build_status
434 self.putChild('source_stamp',
435 SourceStampJsonResource(status,
436 build_status.getSourceStamp()))
437 self.putChild('steps', BuildStepsJsonResource(status, build_status))
438
441
444 help = """All the builds that were run on a builder.
445 """
446 pageTitle = 'AllBuilds'
447
451
453 # Dynamic childs.
454 if isinstance(path, int) or _IS_INT.match(path):
455 build_status = self.builder_status.getBuild(int(path))
456 if build_status:
457 build_status_number = str(build_status.getNumber())
458 # Happens with negative numbers.
459 child = self.children.get(build_status_number)
460 if child:
461 return child
462 # Create it on-demand.
463 child = BuildJsonResource(self.status, build_status)
464 # Cache it. Never cache negative numbers.
465 # TODO(maruel): Cleanup the cache once it's too heavy!
466 self.putChild(build_status_number, child)
467 return child
468 return JsonResource.getChild(self, path, request)
469
471 results = {}
472 # If max > buildCacheSize, it'll trash the cache...
473 cache_size = self.builder_status.master.config.caches['Builds']
474 max = int(RequestArg(request, 'max', cache_size))
475 for i in range(0, max):
476 child = self.getChildWithDefault(-i, request)
477 if not isinstance(child, BuildJsonResource):
478 continue
479 results[child.build_status.getNumber()] = child.asDict(request)
480 return results
481
484 help = """Builds that were run on a builder.
485 """
486 pageTitle = 'Builds'
487
489 AllBuildsJsonResource.__init__(self, status, builder_status)
490 self.putChild('_all', AllBuildsJsonResource(status, builder_status))
491
493 # Transparently redirects to _all if path is not ''.
494 return self.children['_all'].getChildWithDefault(path, request)
495
497 # This would load all the pickles and is way too heavy, especially that
498 # it would trash the cache:
499 # self.children['builds'].asDict(request)
500 # TODO(maruel) This list should also need to be cached but how?
501 builds = dict([
502 (int(file), None)
503 for file in os.listdir(self.builder_status.basedir)
504 if _IS_INT.match(file)
505 ])
506 return builds
507
510 help = """A single build step.
511 """
512 pageTitle = 'BuildStep'
513
515 # buildbot.status.buildstep.BuildStepStatus
516 JsonResource.__init__(self, status)
517 self.build_step_status = build_step_status
518 # TODO self.putChild('logs', LogsJsonResource())
519
521 return self.build_step_status.asDict()
522
525 help = """A list of build steps that occurred during a build.
526 """
527 pageTitle = 'BuildSteps'
528
532 # The build steps are constantly changing until the build is done so
533 # keep a reference to build_status instead
534
536 # Dynamic childs.
537 build_step_status = None
538 if isinstance(path, int) or _IS_INT.match(path):
539 build_step_status = self.build_status.getSteps()[int(path)]
540 else:
541 steps_dict = dict([(step.getName(), step)
542 for step in self.build_status.getSteps()])
543 build_step_status = steps_dict.get(path)
544 if build_step_status:
545 # Create it on-demand.
546 child = BuildStepJsonResource(self.status, build_step_status)
547 # Cache it.
548 index = self.build_status.getSteps().index(build_step_status)
549 self.putChild(str(index), child)
550 self.putChild(build_step_status.getName(), child)
551 return child
552 return JsonResource.getChild(self, path, request)
553
562
565 help = """Describe a single change that originates from a change source.
566 """
567 pageTitle = 'Change'
568
570 # buildbot.changes.changes.Change
571 JsonResource.__init__(self, status)
572 self.change = change
573
575 return self.change.asDict()
576
579 help = """List of changes.
580 """
581 pageTitle = 'Changes'
582
584 JsonResource.__init__(self, status)
585 for c in changes:
586 # c.number can be None or clash another change if the change was
587 # generated inside buildbot or if using multiple pollers.
588 if c.number is not None and str(c.number) not in self.children:
589 self.putChild(str(c.number), ChangeJsonResource(status, c))
590 else:
591 # Temporary hack since it creates information exposure.
592 self.putChild(str(id(c)), ChangeJsonResource(status, c))
593
595 """Don't throw an exception when there is no child."""
596 if not self.children:
597 return {}
598 return JsonResource.asDict(self, request)
599
602 help = """Describe a change source.
603 """
604 pageTitle = 'ChangeSources'
605
607 result = {}
608 n = 0
609 for c in self.status.getChangeSources():
610 # buildbot.changes.changes.ChangeMaster
611 change = {}
612 change['description'] = c.describe()
613 result[n] = change
614 n += 1
615 return result
616
625
628 help = """Describe a slave.
629 """
630 pageTitle = 'Slave'
631
633 JsonResource.__init__(self, status)
634 self.slave_status = slave_status
635 self.name = self.slave_status.getName()
636 self.builders = None
637
639 if self.builders is None:
640 # Figure out all the builders to which it's attached
641 self.builders = []
642 for builderName in self.status.getBuilderNames():
643 if self.name in self.status.getBuilder(builderName).slavenames:
644 self.builders.append(builderName)
645 return self.builders
646
648 results = self.slave_status.asDict()
649 # Enhance it by adding more informations.
650 results['builders'] = {}
651 for builderName in self.getBuilders():
652 builds = []
653 builder_status = self.status.getBuilder(builderName)
654 cache_size = builder_status.master.config.caches['Builds']
655 numbuilds = int(request.args.get('numbuilds', [cache_size - 1])[0])
656 for i in range(1, numbuilds):
657 build_status = builder_status.getBuild(-i)
658 if not build_status or not build_status.isFinished():
659 # If not finished, it will appear in runningBuilds.
660 break
661 if build_status.getSlavename() == self.name:
662 builds.append(build_status.getNumber())
663 results['builders'][builderName] = builds
664 return results
665
668 help = """List the registered slaves.
669 """
670 pageTitle = 'Slaves'
671
673 JsonResource.__init__(self, status)
674 for slave_name in status.getSlaveNames():
675 self.putChild(slave_name,
676 SlaveJsonResource(status,
677 status.getSlave(slave_name)))
678
681 help = """Describe the sources for a SourceStamp.
682 """
683 pageTitle = 'SourceStamp'
684
686 # buildbot.sourcestamp.SourceStamp
687 JsonResource.__init__(self, status)
688 self.source_stamp = source_stamp
689 self.putChild('changes',
690 ChangesJsonResource(status, source_stamp.changes))
691 # TODO(maruel): Should redirect to the patch's url instead.
692 #if source_stamp.patch:
693 # self.putChild('patch', StaticHTML(source_stamp.path))
694
696 return self.source_stamp.asDict()
697
710
714 """Retrieves all json data."""
715 help = """JSON status
716
717 Root page to give a fair amount of information in the current buildbot master
718 status. You may want to use a child instead to reduce the load on the server.
719
720 For help on any sub directory, use url /child/help
721 """
722 pageTitle = 'Buildbot JSON'
723
725 JsonResource.__init__(self, status)
726 self.level = 1
727 self.putChild('builders', BuildersJsonResource(status))
728 self.putChild('change_sources', ChangeSourcesJsonResource(status))
729 self.putChild('project', ProjectJsonResource(status))
730 self.putChild('slaves', SlavesJsonResource(status))
731 self.putChild('metrics', MetricsJsonResource(status))
732 # This needs to be called before the first HelpResource().body call.
733 self.hackExamples()
734
736 result = JsonResource.content(self, request)
737 # This is done to hook the downloaded filename.
738 request.path = 'buildbot'
739 return result
740
742 global EXAMPLES
743 # Find the first builder with a previous build or select the last one.
744 builder = None
745 for b in self.status.getBuilderNames():
746 builder = self.status.getBuilder(b)
747 if builder.getBuild(-1):
748 break
749 if not builder:
750 return
751 EXAMPLES = EXAMPLES.replace('<A_BUILDER>', builder.getName())
752 build = builder.getBuild(-1)
753 if build:
754 EXAMPLES = EXAMPLES.replace('<A_BUILD>', str(build.getNumber()))
755 if builder.slavenames:
756 EXAMPLES = EXAMPLES.replace('<A_SLAVE>', builder.slavenames[0])
757
758 # vim: set ts=4 sts=4 sw=4 et:
759
| Trees | Indices | Help |
|
|---|
| Generated by Epydoc 3.0.1 on Sun Mar 25 19:40:45 2012 | http://epydoc.sourceforge.net |