| Trees | Indices | Help |
|
|---|
|
|
1 # This file is part of Buildbot. Buildbot is free software: you can
2 # redistribute it and/or modify it under the terms of the GNU General Public
3 # License as published by the Free Software Foundation, version 2.
4 #
5 # This program is distributed in the hope that it will be useful, but WITHOUT
6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
8 # details.
9 #
10 # You should have received a copy of the GNU General Public License along with
11 # this program; if not, write to the Free Software Foundation, Inc., 51
12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
13 #
14 # Portions Copyright Buildbot Team Members
15 # Original Copyright (c) 2010 The Chromium Authors.
16
17 """Simple JSON exporter."""
18
19 import datetime
20 import os
21 import re
22
23 from twisted.internet import defer
24 from twisted.web import html, resource, server
25
26 from buildbot.status.web.base import HtmlResource
27 from buildbot.util import json
28
29
30 _IS_INT = re.compile('^[-+]?\d+$')
31
32
33 FLAGS = """\
34 - as_text
35 - By default, application/json is used. Setting as_text=1 change the type
36 to text/plain and implicitly sets compact=0 and filter=1. Mainly useful to
37 look at the result in a web browser.
38 - compact
39 - By default, the json data is compact and defaults to 1. For easier to read
40 indented output, set compact=0.
41 - select
42 - By default, most children data is listed. You can do a random selection
43 of data by using select=<sub-url> multiple times to coagulate data.
44 "select=" includes the actual url otherwise it is skipped.
45 - filter
46 - Filters out null, false, and empty string, list and dict. This reduce the
47 amount of useless data sent.
48 - callback
49 - Enable uses of JSONP as described in
50 http://en.wikipedia.org/wiki/JSONP. Note that
51 Access-Control-Allow-Origin:* is set in the HTTP response header so you
52 can use this in compatible browsers.
53 """
54
55 EXAMPLES = """\
56 - /json
57 - Root node, that *doesn't* mean all the data. Many things (like logs) must
58 be explicitly queried for performance reasons.
59 - /json/builders/
60 - All builders.
61 - /json/builders/<A_BUILDER>
62 - A specific builder as compact text.
63 - /json/builders/<A_BUILDER>/builds
64 - All *cached* builds.
65 - /json/builders/<A_BUILDER>/builds/_all
66 - All builds. Warning, reads all previous build data.
67 - /json/builders/<A_BUILDER>/builds/<A_BUILD>
68 - Where <A_BUILD> is either positive, a build number, or negative, a past
69 build.
70 - /json/builders/<A_BUILDER>/builds/-1/source_stamp/changes
71 - Build changes
72 - /json/builders/<A_BUILDER>/builds?select=-1&select=-2
73 - Two last builds on '<A_BUILDER>' builder.
74 - /json/builders/<A_BUILDER>/builds?select=-1/source_stamp/changes&select=-2/source_stamp/changes
75 - Changes of the two last builds on '<A_BUILDER>' builder.
76 - /json/builders/<A_BUILDER>/slaves
77 - Slaves associated to this builder.
78 - /json/builders/<A_BUILDER>?select=&select=slaves
79 - Builder information plus details information about its slaves. Neat eh?
80 - /json/slaves/<A_SLAVE>
81 - A specific slave.
82 - /json?select=slaves/<A_SLAVE>/&select=project&select=builders/<A_BUILDER>/builds/<A_BUILD>
83 - A selection of random unrelated stuff as an random example. :)
84 """
88 return request.args.get(arg, [default])[0]
89
92 value = RequestArg(request, arg, default)
93 if value in (False, True):
94 return value
95 value = value.lower()
96 if value in ('1', 'true'):
97 return True
98 if value in ('0', 'false'):
99 return False
100 # Ignore value.
101 return default
102
105 """Returns a copy with None, False, "", [], () and {} removed.
106 Warning: converts tuple to list."""
107 if isinstance(data, (list, tuple)):
108 # Recurse in every items and filter them out.
109 items = map(FilterOut, data)
110 if not filter(lambda x: not x in ('', False, None, [], {}, ()), items):
111 return None
112 return items
113 elif isinstance(data, dict):
114 return dict(filter(lambda x: not x[1] in ('', False, None, [], {}, ()),
115 [(k, FilterOut(v)) for (k, v) in data.iteritems()]))
116 else:
117 return data
118
121 """Base class for json data."""
122
123 contentType = "application/json"
124 cache_seconds = 60
125 help = None
126 pageTitle = None
127 level = 0
128
130 """Adds transparent lazy-child initialization."""
131 resource.Resource.__init__(self)
132 # buildbot.status.builder.Status
133 self.status = status
134 if self.help:
135 pageTitle = ''
136 if self.pageTitle:
137 pageTitle = self.pageTitle + ' help'
138 self.putChild('help',
139 HelpResource(self.help, pageTitle=pageTitle, parent_node=self))
140
142 """Adds transparent support for url ending with /"""
143 if path == "" and len(request.postpath) == 0:
144 return self
145 # Equivalent to resource.Resource.getChildWithDefault()
146 if self.children.has_key(path):
147 return self.children[path]
148 return self.getChild(path, request)
149
151 """Adds the resource's level for help links generation."""
152
153 def RecurseFix(res, level):
154 res.level = level + 1
155 for c in res.children.itervalues():
156 RecurseFix(c, res.level)
157
158 RecurseFix(res, self.level)
159 resource.Resource.putChild(self, name, res)
160
162 """Renders a HTTP GET at the http request level."""
163 d = defer.maybeDeferred(lambda : self.content(request))
164 def handle(data):
165 if isinstance(data, unicode):
166 data = data.encode("utf-8")
167 request.setHeader("Access-Control-Allow-Origin", "*")
168 if RequestArgToBool(request, 'as_text', False):
169 request.setHeader("content-type", 'text/plain')
170 else:
171 request.setHeader("content-type", self.contentType)
172 request.setHeader("content-disposition",
173 "attachment; filename=\"%s.json\"" % request.path)
174 # Make sure we get fresh pages.
175 if self.cache_seconds:
176 now = datetime.datetime.utcnow()
177 expires = now + datetime.timedelta(seconds=self.cache_seconds)
178 request.setHeader("Expires",
179 expires.strftime("%a, %d %b %Y %H:%M:%S GMT"))
180 request.setHeader("Pragma", "no-cache")
181 return data
182 d.addCallback(handle)
183 def ok(data):
184 request.write(data)
185 request.finish()
186 def fail(f):
187 request.processingFailed(f)
188 return None # processingFailed will log this for us
189 d.addCallbacks(ok, fail)
190 return server.NOT_DONE_YET
191
192 @defer.deferredGenerator
194 """Renders the json dictionaries."""
195 # Supported flags.
196 select = request.args.get('select')
197 as_text = RequestArgToBool(request, 'as_text', False)
198 filter_out = RequestArgToBool(request, 'filter', as_text)
199 compact = RequestArgToBool(request, 'compact', not as_text)
200 callback = request.args.get('callback')
201
202 # Implement filtering at global level and every child.
203 if select is not None:
204 del request.args['select']
205 # Do not render self.asDict()!
206 data = {}
207 # Remove superfluous /
208 select = [s.strip('/') for s in select]
209 select.sort(cmp=lambda x,y: cmp(x.count('/'), y.count('/')),
210 reverse=True)
211 for item in select:
212 # Start back at root.
213 node = data
214 # Implementation similar to twisted.web.resource.getChildForRequest
215 # but with a hacked up request.
216 child = self
217 prepath = request.prepath[:]
218 postpath = request.postpath[:]
219 request.postpath = filter(None, item.split('/'))
220 while request.postpath and not child.isLeaf:
221 pathElement = request.postpath.pop(0)
222 node[pathElement] = {}
223 node = node[pathElement]
224 request.prepath.append(pathElement)
225 child = child.getChildWithDefault(pathElement, request)
226
227 # some asDict methods return a Deferred, so handle that
228 # properly
229 if hasattr(child, 'asDict'):
230 wfd = defer.waitForDeferred(
231 defer.maybeDeferred(lambda :
232 child.asDict(request)))
233 yield wfd
234 child_dict = wfd.getResult()
235 else:
236 child_dict = {
237 'error' : 'Not available',
238 }
239 node.update(child_dict)
240
241 request.prepath = prepath
242 request.postpath = postpath
243 else:
244 wfd = defer.waitForDeferred(
245 defer.maybeDeferred(lambda :
246 self.asDict(request)))
247 yield wfd
248 data = wfd.getResult()
249
250 if filter_out:
251 data = FilterOut(data)
252 if compact:
253 data = json.dumps(data, sort_keys=True, separators=(',',':'))
254 else:
255 data = json.dumps(data, sort_keys=True, indent=2)
256 if callback:
257 # Only accept things that look like identifiers for now
258 callback = callback[0]
259 if re.match(r'^[a-zA-Z$][a-zA-Z$0-9.]*$', callback):
260 data = '%s(%s);' % (callback, data)
261 yield data
262
263 @defer.deferredGenerator
265 """Generates the json dictionary.
266
267 By default, renders every childs."""
268 if self.children:
269 data = {}
270 for name in self.children:
271 child = self.getChildWithDefault(name, request)
272 if isinstance(child, JsonResource):
273 wfd = defer.waitForDeferred(
274 defer.maybeDeferred(lambda :
275 child.asDict(request)))
276 yield wfd
277 data[name] = wfd.getResult()
278 # else silently pass over non-json resources.
279 yield data
280 else:
281 raise NotImplementedError()
282
285 """Convert a string in a wiki-style format into HTML."""
286 indent = 0
287 in_item = False
288 output = []
289 for line in text.splitlines(False):
290 match = re.match(r'^( +)\- (.*)$', line)
291 if match:
292 if indent < len(match.group(1)):
293 output.append('<ul>')
294 indent = len(match.group(1))
295 elif indent > len(match.group(1)):
296 while indent > len(match.group(1)):
297 output.append('</ul>')
298 indent -= 2
299 if in_item:
300 # Close previous item
301 output.append('</li>')
302 output.append('<li>')
303 in_item = True
304 line = match.group(2)
305 elif indent:
306 if line.startswith((' ' * indent) + ' '):
307 # List continuation
308 line = line.strip()
309 else:
310 # List is done
311 if in_item:
312 output.append('</li>')
313 in_item = False
314 while indent > 0:
315 output.append('</ul>')
316 indent -= 2
317
318 if line.startswith('/'):
319 if not '?' in line:
320 line_full = line + '?as_text=1'
321 else:
322 line_full = line + '&as_text=1'
323 output.append('<a href="' + html.escape(line_full) + '">' +
324 html.escape(line) + '</a>')
325 else:
326 output.append(html.escape(line).replace(' ', ' '))
327 if not in_item:
328 output.append('<br>')
329
330 if in_item:
331 output.append('</li>')
332 while indent > 0:
333 output.append('</ul>')
334 indent -= 2
335 return '\n'.join(output)
336
340 HtmlResource.__init__(self)
341 self.text = text
342 self.pageTitle = pageTitle
343 self.parent_node = parent_node
344
346 cxt['level'] = self.parent_node.level
347 cxt['text'] = ToHtml(self.text)
348 cxt['children'] = [ n for n in self.parent_node.children.keys() if n != 'help' ]
349 cxt['flags'] = ToHtml(FLAGS)
350 cxt['examples'] = ToHtml(EXAMPLES).replace(
351 'href="/json',
352 'href="%sjson' % (self.level * '../'))
353
354 template = request.site.buildbot_service.templates.get_template("jsonhelp.html")
355 return template.render(**cxt)
356
358 help = """Describe pending builds for a builder.
359 """
360 pageTitle = 'Builder'
361
365
367 # buildbot.status.builder.BuilderStatus
368 d = self.builder_status.getPendingBuildRequestStatuses()
369 def to_dict(statuses):
370 return defer.gatherResults(
371 [ b.asDict_async() for b in statuses ])
372 d.addCallback(to_dict)
373 return d
374
377 help = """Describe a single builder.
378 """
379 pageTitle = 'Builder'
380
382 JsonResource.__init__(self, status)
383 self.builder_status = builder_status
384 self.putChild('builds', BuildsJsonResource(status, builder_status))
385 self.putChild('slaves', BuilderSlavesJsonResources(status,
386 builder_status))
387 self.putChild(
388 'pendingBuilds',
389 BuilderPendingBuildsJsonResource(status, builder_status))
390
394
397 help = """List of all the builders defined on a master.
398 """
399 pageTitle = 'Builders'
400
402 JsonResource.__init__(self, status)
403 for builder_name in self.status.getBuilderNames():
404 self.putChild(builder_name,
405 BuilderJsonResource(status,
406 status.getBuilder(builder_name)))
407
410 help = """Describe the slaves attached to a single builder.
411 """
412 pageTitle = 'BuilderSlaves'
413
415 JsonResource.__init__(self, status)
416 self.builder_status = builder_status
417 for slave_name in self.builder_status.slavenames:
418 self.putChild(slave_name,
419 SlaveJsonResource(status,
420 self.status.getSlave(slave_name)))
421
424 help = """Describe a single build.
425 """
426 pageTitle = 'Build'
427
429 JsonResource.__init__(self, status)
430 self.build_status = build_status
431 self.putChild('source_stamp',
432 SourceStampJsonResource(status,
433 build_status.getSourceStamp()))
434 self.putChild('steps', BuildStepsJsonResource(status, build_status))
435
438
441 help = """All the builds that were run on a builder.
442 """
443 pageTitle = 'AllBuilds'
444
448
450 # Dynamic childs.
451 if isinstance(path, int) or _IS_INT.match(path):
452 build_status = self.builder_status.getBuild(int(path))
453 if build_status:
454 build_status_number = str(build_status.getNumber())
455 # Happens with negative numbers.
456 child = self.children.get(build_status_number)
457 if child:
458 return child
459 # Create it on-demand.
460 child = BuildJsonResource(self.status, build_status)
461 # Cache it. Never cache negative numbers.
462 # TODO(maruel): Cleanup the cache once it's too heavy!
463 self.putChild(build_status_number, child)
464 return child
465 return JsonResource.getChild(self, path, request)
466
468 results = {}
469 # If max > buildCacheSize, it'll trash the cache...
470 max = int(RequestArg(request, 'max',
471 self.builder_status.buildCacheSize))
472 for i in range(0, max):
473 child = self.getChildWithDefault(-i, request)
474 if not isinstance(child, BuildJsonResource):
475 continue
476 results[child.build_status.getNumber()] = child.asDict(request)
477 return results
478
481 help = """Builds that were run on a builder.
482 """
483 pageTitle = 'Builds'
484
486 AllBuildsJsonResource.__init__(self, status, builder_status)
487 self.putChild('_all', AllBuildsJsonResource(status, builder_status))
488
490 # Transparently redirects to _all if path is not ''.
491 return self.children['_all'].getChildWithDefault(path, request)
492
494 # This would load all the pickles and is way too heavy, especially that
495 # it would trash the cache:
496 # self.children['builds'].asDict(request)
497 # TODO(maruel) This list should also need to be cached but how?
498 builds = dict([
499 (int(file), None)
500 for file in os.listdir(self.builder_status.basedir)
501 if _IS_INT.match(file)
502 ])
503 return builds
504
507 help = """A single build step.
508 """
509 pageTitle = 'BuildStep'
510
512 # buildbot.status.buildstep.BuildStepStatus
513 JsonResource.__init__(self, status)
514 self.build_step_status = build_step_status
515 # TODO self.putChild('logs', LogsJsonResource())
516
518 return self.build_step_status.asDict()
519
522 help = """A list of build steps that occurred during a build.
523 """
524 pageTitle = 'BuildSteps'
525
529 # The build steps are constantly changing until the build is done so
530 # keep a reference to build_status instead
531
533 # Dynamic childs.
534 build_step_status = None
535 if isinstance(path, int) or _IS_INT.match(path):
536 build_step_status = self.build_status.getSteps()[int(path)]
537 else:
538 steps_dict = dict([(step.getName(), step)
539 for step in self.build_status.getSteps()])
540 build_step_status = steps_dict.get(path)
541 if build_step_status:
542 # Create it on-demand.
543 child = BuildStepJsonResource(self.status, build_step_status)
544 # Cache it.
545 index = self.build_status.getSteps().index(build_step_status)
546 self.putChild(str(index), child)
547 self.putChild(build_step_status.getName(), child)
548 return child
549 return JsonResource.getChild(self, path, request)
550
559
562 help = """Describe a single change that originates from a change source.
563 """
564 pageTitle = 'Change'
565
567 # buildbot.changes.changes.Change
568 JsonResource.__init__(self, status)
569 self.change = change
570
572 return self.change.asDict()
573
576 help = """List of changes.
577 """
578 pageTitle = 'Changes'
579
581 JsonResource.__init__(self, status)
582 for c in changes:
583 # c.number can be None or clash another change if the change was
584 # generated inside buildbot or if using multiple pollers.
585 if c.number is not None and str(c.number) not in self.children:
586 self.putChild(str(c.number), ChangeJsonResource(status, c))
587 else:
588 # Temporary hack since it creates information exposure.
589 self.putChild(str(id(c)), ChangeJsonResource(status, c))
590
592 """Don't throw an exception when there is no child."""
593 if not self.children:
594 return {}
595 return JsonResource.asDict(self, request)
596
599 help = """Describe a change source.
600 """
601 pageTitle = 'ChangeSources'
602
604 result = {}
605 n = 0
606 for c in self.status.getChangeSources():
607 # buildbot.changes.changes.ChangeMaster
608 change = {}
609 change['description'] = c.describe()
610 result[n] = change
611 n += 1
612 return result
613
622
625 help = """Describe a slave.
626 """
627 pageTitle = 'Slave'
628
630 JsonResource.__init__(self, status)
631 self.slave_status = slave_status
632 self.name = self.slave_status.getName()
633 self.builders = None
634
636 if self.builders is None:
637 # Figure out all the builders to which it's attached
638 self.builders = []
639 for builderName in self.status.getBuilderNames():
640 if self.name in self.status.getBuilder(builderName).slavenames:
641 self.builders.append(builderName)
642 return self.builders
643
645 results = self.slave_status.asDict()
646 # Enhance it by adding more informations.
647 results['builders'] = {}
648 for builderName in self.getBuilders():
649 builds = []
650 builder_status = self.status.getBuilder(builderName)
651 for i in range(1, builder_status.buildCacheSize - 1):
652 build_status = builder_status.getBuild(-i)
653 if not build_status or not build_status.isFinished():
654 # If not finished, it will appear in runningBuilds.
655 break
656 if build_status.getSlavename() == self.name:
657 builds.append(build_status.getNumber())
658 results['builders'][builderName] = builds
659 return results
660
663 help = """List the registered slaves.
664 """
665 pageTitle = 'Slaves'
666
668 JsonResource.__init__(self, status)
669 for slave_name in status.getSlaveNames():
670 self.putChild(slave_name,
671 SlaveJsonResource(status,
672 status.getSlave(slave_name)))
673
676 help = """Describe the sources for a SourceStamp.
677 """
678 pageTitle = 'SourceStamp'
679
681 # buildbot.sourcestamp.SourceStamp
682 JsonResource.__init__(self, status)
683 self.source_stamp = source_stamp
684 self.putChild('changes',
685 ChangesJsonResource(status, source_stamp.changes))
686 # TODO(maruel): Should redirect to the patch's url instead.
687 #if source_stamp.patch:
688 # self.putChild('patch', StaticHTML(source_stamp.path))
689
691 return self.source_stamp.asDict()
692
705
709 """Retrieves all json data."""
710 help = """JSON status
711
712 Root page to give a fair amount of information in the current buildbot master
713 status. You may want to use a child instead to reduce the load on the server.
714
715 For help on any sub directory, use url /child/help
716 """
717 pageTitle = 'Buildbot JSON'
718
720 JsonResource.__init__(self, status)
721 self.level = 1
722 self.putChild('builders', BuildersJsonResource(status))
723 self.putChild('change_sources', ChangeSourcesJsonResource(status))
724 self.putChild('project', ProjectJsonResource(status))
725 self.putChild('slaves', SlavesJsonResource(status))
726 self.putChild('metrics', MetricsJsonResource(status))
727 # This needs to be called before the first HelpResource().body call.
728 self.hackExamples()
729
731 result = JsonResource.content(self, request)
732 # This is done to hook the downloaded filename.
733 request.path = 'buildbot'
734 return result
735
737 global EXAMPLES
738 # Find the first builder with a previous build or select the last one.
739 builder = None
740 for b in self.status.getBuilderNames():
741 builder = self.status.getBuilder(b)
742 if builder.getBuild(-1):
743 break
744 if not builder:
745 return
746 EXAMPLES = EXAMPLES.replace('<A_BUILDER>', builder.getName())
747 build = builder.getBuild(-1)
748 if build:
749 EXAMPLES = EXAMPLES.replace('<A_BUILD>', str(build.getNumber()))
750 if builder.slavenames:
751 EXAMPLES = EXAMPLES.replace('<A_SLAVE>', builder.slavenames[0])
752
753 # vim: set ts=4 sts=4 sw=4 et:
754
| Trees | Indices | Help |
|
|---|
| Generated by Epydoc 3.0.1 on Sun Jul 17 13:45:29 2011 | http://epydoc.sourceforge.net |