| Home | Trees | Indices | Help |
|
|---|
|
|
1 # -*- test-case-name: buildbot.test.test_web_status_json -*-
2 # Original Copyright (c) 2010 The Chromium Authors.
3
4 """Simple JSON exporter."""
5
6 import datetime
7 import os
8 import re
9
10 from twisted.web import error, html, resource
11
12 from buildbot.status.web.base import HtmlResource
13 from buildbot.util import json
14
15
16 _IS_INT = re.compile('^[-+]?\d+$')
17
18
19 FLAGS = """\
20 - as_text
21 - By default, application/json is used. Setting as_text=1 change the type
22 to text/plain and implicitly sets compact=0 and filter=1. Mainly useful to
23 look at the result in a web browser.
24 - compact
25 - By default, the json data is compact and defaults to 1. For easier to read
26 indented output, set compact=0.
27 - select
28 - By default, most children data is listed. You can do a random selection
29 of data by using select=<sub-url> multiple times to coagulate data.
30 "select=" includes the actual url otherwise it is skipped.
31 - filter
32 - Filters out null, false, and empty string, list and dict. This reduce the
33 amount of useless data sent.
34
35 """
36
37 EXAMPLES = """\
38 - /json
39 - Root node, that *doesn't* mean all the data. Many things (like logs) must
40 be explicitly queried for performance reasons.
41 - /json/builders/
42 - All builders.
43 - /json/builders/<A_BUILDER>
44 - A specific builder as compact text.
45 - /json/builders/<A_BUILDER>/builds
46 - All *cached* builds.
47 - /json/builders/<A_BUILDER>/builds/_all
48 - All builds. Warning, reads all previous build data.
49 - /json/builders/<A_BUILDER>/builds/<A_BUILD>
50 - Where <A_BUILD> is either positive, a build number, or negative, a past
51 build.
52 - /json/builders/<A_BUILDER>/builds/-1/source_stamp/changes
53 - Build changes
54 - /json/builders/<A_BUILDER>/builds?select=-1&select=-2
55 - Two last builds on '<A_BUILDER>' builder.
56 - /json/builders/<A_BUILDER>/builds?select=-1/source_stamp/changes&select=-2/source_stamp/changes
57 - Changes of the two last builds on '<A_BUILDER>' builder.
58 - /json/builders/<A_BUILDER>/slaves
59 - Slaves associated to this builder.
60 - /json/builders/<A_BUILDER>?select=&select=slaves
61 - Builder information plus details information about its slaves. Neat eh?
62 - /json/slaves/<A_SLAVE>
63 - A specific slave.
64 - /json?select=slaves/<A_SLAVE>/&select=project&select=builders/<A_BUILDER>/builds/<A_BUILD>
65 - A selection of random unrelated stuff as an random example. :)
66 """
67
68
70 return request.args.get(arg, [default])[0]
71
72
74 value = RequestArg(request, arg, default)
75 if value in (False, True):
76 return value
77 value = value.lower()
78 if value in ('1', 'true'):
79 return True
80 if value in ('0', 'false'):
81 return False
82 # Ignore value.
83 return default
84
85
87 """Additional method for twisted.web.error.Error."""
88 result = {}
89 result['http_error'] = self.status
90 result['response'] = self.response
91 return result
92
93
95 """Additional method for twisted.web.error.Error."""
96 result = {}
97 result['http_error'] = self.code
98 result['response'] = self.brief
99 result['detail'] = self.detail
100 return result
101
102
103 # Add .asDict() method to twisted.web.error.Error to simplify the code below.
104 error.Error.asDict = TwistedWebErrorAsDict
105 error.PageRedirect.asDict = TwistedWebErrorAsDict
106 error.ErrorPage.asDict = TwistedWebErrorPageAsDict
107 error.NoResource.asDict = TwistedWebErrorPageAsDict
108 error.ForbiddenResource.asDict = TwistedWebErrorPageAsDict
109
110
112 """Returns a copy with None, False, "", [], () and {} removed.
113 Warning: converts tuple to list."""
114 if isinstance(data, (list, tuple)):
115 # Recurse in every items and filter them out.
116 items = map(FilterOut, data)
117 if not filter(lambda x: not x in ('', False, None, [], {}, ()), items):
118 return None
119 return items
120 elif isinstance(data, dict):
121 return dict(filter(lambda x: not x[1] in ('', False, None, [], {}, ()),
122 [(k, FilterOut(v)) for (k, v) in data.iteritems()]))
123 else:
124 return data
125
126
128 """Base class for json data."""
129
130 contentType = "application/json"
131 cache_seconds = 60
132 help = None
133 title = None
134 level = 0
135
137 """Adds transparent lazy-child initialization."""
138 resource.Resource.__init__(self)
139 # buildbot.status.builder.Status
140 self.status = status
141 if self.help:
142 title = ''
143 if self.title:
144 title = self.title + ' help'
145 self.putChild('help',
146 HelpResource(self.help, title=title, parent_node=self))
147
149 """Adds transparent support for url ending with /"""
150 if path == "" and len(request.postpath) == 0:
151 return self
152 # Equivalent to resource.Resource.getChildWithDefault()
153 if self.children.has_key(path):
154 return self.children[path]
155 return self.getChild(path, request)
156
158 """Adds the resource's level for help links generation."""
159
160 def RecurseFix(res, level):
161 res.level = level + 1
162 for c in res.children.itervalues():
163 RecurseFix(c, res.level)
164
165 RecurseFix(res, self.level)
166 resource.Resource.putChild(self, name, res)
167
169 """Renders a HTTP GET at the http request level."""
170 data = self.content(request)
171 if isinstance(data, unicode):
172 data = data.encode("utf-8")
173 if RequestArgToBool(request, 'as_text', False):
174 request.setHeader("content-type", 'text/plain')
175 else:
176 request.setHeader("content-type", self.contentType)
177 request.setHeader("content-disposition",
178 "attachment; filename=\"%s.json\"" % request.path)
179 # Make sure we get fresh pages.
180 if self.cache_seconds:
181 now = datetime.datetime.utcnow()
182 expires = now + datetime.timedelta(seconds=self.cache_seconds)
183 request.setHeader("Expires",
184 expires.strftime("%a, %d %b %Y %H:%M:%S GMT"))
185 request.setHeader("Pragma", "no-cache")
186 return data
187
189 """Renders the json dictionaries."""
190 # Implement filtering at global level and every child.
191 select = request.args.get('select')
192 if select is not None:
193 del request.args['select']
194 # Do not render self.asDict()!
195 data = {}
196 # Remove superfluous /
197 select = [s.strip('/') for s in select]
198 select.sort(cmp=lambda x,y: cmp(x.count('/'), y.count('/')),
199 reverse=True)
200 for item in select:
201 # Start back at root.
202 node = data
203 # Implementation similar to twisted.web.resource.getChildForRequest
204 # but with a hacked up request.
205 child = self
206 prepath = request.prepath[:]
207 postpath = request.postpath[:]
208 request.postpath = filter(None, item.split('/'))
209 while request.postpath and not child.isLeaf:
210 pathElement = request.postpath.pop(0)
211 node[pathElement] = {}
212 node = node[pathElement]
213 request.prepath.append(pathElement)
214 child = child.getChildWithDefault(pathElement, request)
215 node.update(child.asDict(request))
216 request.prepath = prepath
217 request.postpath = postpath
218 else:
219 data = self.asDict(request)
220 as_text = RequestArgToBool(request, 'as_text', False)
221 filter_out = RequestArgToBool(request, 'filter', as_text)
222 if filter_out:
223 data = FilterOut(data)
224 if RequestArgToBool(request, 'compact', not as_text):
225 return json.dumps(data, sort_keys=True, separators=(',',':'))
226 else:
227 return json.dumps(data, sort_keys=True, indent=2)
228
230 """Generates the json dictionary.
231
232 By default, renders every childs."""
233 if self.children:
234 data = {}
235 for name in self.children:
236 child = self.getChildWithDefault(name, request)
237 if isinstance(child, JsonResource):
238 data[name] = child.asDict(request)
239 # else silently pass over non-json resources.
240 return data
241 else:
242 raise NotImplementedError()
243
244
246 """Convert a string in a wiki-style format into HTML."""
247 indent = 0
248 in_item = False
249 output = []
250 for line in text.splitlines(False):
251 match = re.match(r'^( +)\- (.*)$', line)
252 if match:
253 if indent < len(match.group(1)):
254 output.append('<ul>')
255 indent = len(match.group(1))
256 elif indent > len(match.group(1)):
257 while indent > len(match.group(1)):
258 output.append('</ul>')
259 indent -= 2
260 if in_item:
261 # Close previous item
262 output.append('</li>')
263 output.append('<li>')
264 in_item = True
265 line = match.group(2)
266 elif indent:
267 if line.startswith((' ' * indent) + ' '):
268 # List continuation
269 line = line.strip()
270 else:
271 # List is done
272 if in_item:
273 output.append('</li>')
274 in_item = False
275 while indent > 0:
276 output.append('</ul>')
277 indent -= 2
278
279 if line.startswith('/'):
280 if not '?' in line:
281 line_full = line + '?as_text=1'
282 else:
283 line_full = line + '&as_text=1'
284 output.append('<a href="' + html.escape(line_full) + '">' +
285 html.escape(line) + '</a>')
286 else:
287 output.append(html.escape(line).replace(' ', ' '))
288 if not in_item:
289 output.append('<br>')
290
291 if in_item:
292 output.append('</li>')
293 while indent > 0:
294 output.append('</ul>')
295 indent -= 2
296 return '\n'.join(output)
297
298
301 HtmlResource.__init__(self)
302 self.text = text
303 self.title = title
304 self.parent_node = parent_node
305
307 cxt['level'] = self.parent_node.level
308 cxt['text'] = ToHtml(self.text)
309 cxt['children'] = [ n for n in self.parent_node.children.keys() if n != 'help' ]
310 cxt['flags'] = ToHtml(FLAGS)
311 cxt['examples'] = ToHtml(EXAMPLES).replace(
312 'href="/json',
313 'href="%sjson' % (self.level * '../'))
314
315 template = request.site.buildbot_service.templates.get_template("jsonhelp.html")
316 return template.render(**cxt)
317
319 help = """Describe a single builder.
320 """
321 title = 'Builder'
322
324 JsonResource.__init__(self, status)
325 self.builder_status = builder_status
326 self.putChild('builds', BuildsJsonResource(status, builder_status))
327 self.putChild('slaves', BuilderSlavesJsonResources(status,
328 builder_status))
329
333
334
336 help = """List of all the builders defined on a master.
337 """
338 title = 'Builders'
339
341 JsonResource.__init__(self, status)
342 for builder_name in self.status.getBuilderNames():
343 self.putChild(builder_name,
344 BuilderJsonResource(status,
345 status.getBuilder(builder_name)))
346
347
349 help = """Describe the slaves attached to a single builder.
350 """
351 title = 'BuilderSlaves'
352
354 JsonResource.__init__(self, status)
355 self.builder_status = builder_status
356 for slave_name in self.builder_status.slavenames:
357 self.putChild(slave_name,
358 SlaveJsonResource(status,
359 self.status.getSlave(slave_name)))
360
361
363 help = """Describe a single build.
364 """
365 title = 'Build'
366
368 JsonResource.__init__(self, status)
369 self.build_status = build_status
370 self.putChild('source_stamp',
371 SourceStampJsonResource(status,
372 build_status.getSourceStamp()))
373 self.putChild('steps', BuildStepsJsonResource(status, build_status))
374
377
378
380 help = """All the builds that were run on a builder.
381 """
382 title = 'AllBuilds'
383
387
389 # Dynamic childs.
390 if isinstance(path, int) or _IS_INT.match(path):
391 build_status = self.builder_status.getBuild(int(path))
392 if build_status:
393 build_status_number = str(build_status.getNumber())
394 # Happens with negative numbers.
395 child = self.children.get(build_status_number)
396 if child:
397 return child
398 # Create it on-demand.
399 child = BuildJsonResource(self.status, build_status)
400 # Cache it. Never cache negative numbers.
401 # TODO(maruel): Cleanup the cache once it's too heavy!
402 self.putChild(build_status_number, child)
403 return child
404 return JsonResource.getChild(self, path, request)
405
407 results = {}
408 # If max > buildCacheSize, it'll trash the cache...
409 max = int(RequestArg(request, 'max',
410 self.builder_status.buildCacheSize))
411 for i in range(0, max):
412 child = self.getChildWithDefault(-i, request)
413 if not isinstance(child, BuildJsonResource):
414 continue
415 results[child.build_status.getNumber()] = child.asDict(request)
416 return results
417
418
420 help = """Builds that were run on a builder.
421 """
422 title = 'Builds'
423
425 AllBuildsJsonResource.__init__(self, status, builder_status)
426 self.putChild('_all', AllBuildsJsonResource(status, builder_status))
427
429 # Transparently redirects to _all if path is not ''.
430 return self.children['_all'].getChildWithDefault(path, request)
431
433 # This would load all the pickles and is way too heavy, especially that
434 # it would trash the cache:
435 # self.children['builds'].asDict(request)
436 # TODO(maruel) This list should also need to be cached but how?
437 builds = dict([
438 (int(file), None)
439 for file in os.listdir(self.builder_status.basedir)
440 if _IS_INT.match(file)
441 ])
442 return builds
443
444
446 help = """A single build step.
447 """
448 title = 'BuildStep'
449
451 # buildbot.status.builder.BuildStepStatus
452 JsonResource.__init__(self, status)
453 self.build_step_status = build_step_status
454 # TODO self.putChild('logs', LogsJsonResource())
455
457 return self.build_step_status.asDict()
458
459
461 help = """A list of build steps that occurred during a build.
462 """
463 title = 'BuildSteps'
464
468 # The build steps are constantly changing until the build is done so
469 # keep a reference to build_status instead
470
472 # Dynamic childs.
473 build_set_status = None
474 if isinstance(path, int) or _IS_INT.match(path):
475 build_set_status = self.build_status.getSteps[int(path)]
476 else:
477 steps_dict = dict([(step.getName(), step)
478 for step in self.build_status.getStep()])
479 build_set_status = steps_dict.get(path)
480 if build_set_status:
481 # Create it on-demand.
482 child = BuildStepJsonResource(status, build_step_status)
483 # Cache it.
484 index = self.build_status.getSteps().index(build_step_status)
485 self.putChild(str(index), child)
486 self.putChild(build_set_status.getName(), child)
487 return child
488 return JsonResource.getChild(self, path, request)
489
498
499
501 help = """Describe a single change that originates from a change source.
502 """
503 title = 'Change'
504
506 # buildbot.changes.changes.Change
507 JsonResource.__init__(self, status)
508 self.change = change
509
511 return self.change.asDict()
512
513
515 help = """List of changes.
516 """
517 title = 'Changes'
518
520 JsonResource.__init__(self, status)
521 for c in changes:
522 # TODO(maruel): Problem with multiple changes with the same number.
523 # Probably try server hack specific so we could fix it on this side
524 # instead. But there is still the problem with multiple pollers from
525 # different repo where the numbers could clash.
526 number = str(c.number)
527 while number in self.children:
528 # TODO(maruel): Do something better?
529 number = str(int(c.number)+1)
530 self.putChild(number, ChangeJsonResource(status, c))
531
533 """Don't throw an exception when there is no child."""
534 if not self.children:
535 return {}
536 return JsonResource.asDict(self, request)
537
538
540 help = """Describe a change source.
541 """
542 title = 'ChangeSources'
543
545 result = {}
546 n = 0
547 for c in self.status.getChangeSources():
548 # buildbot.changes.changes.ChangeMaster
549 change = {}
550 change['description'] = c.describe()
551 result[n] = change
552 n += 1
553 return result
554
555
563
564
566 help = """Describe a slave.
567 """
568 title = 'Slave'
569
571 JsonResource.__init__(self, status)
572 self.slave_status = slave_status
573 self.name = self.slave_status.getName()
574 self.builders = None
575
577 if self.builders is None:
578 # Figure out all the builders to which it's attached
579 self.builders = []
580 for builderName in self.status.getBuilderNames():
581 if self.name in self.status.getBuilder(builderName).slavenames:
582 self.builders.append(builderName)
583 return self.builders
584
586 results = self.slave_status.asDict()
587 # Enhance it by adding more informations.
588 results['builders'] = {}
589 for builderName in self.getBuilders():
590 builds = []
591 builder_status = self.status.getBuilder(builderName)
592 for i in range(1, builder_status.buildCacheSize - 1):
593 build_status = builder_status.getBuild(-i)
594 if not build_status or not build_status.isFinished():
595 # If not finished, it will appear in runningBuilds.
596 break
597 if build_status.getSlavename() == self.name:
598 builds.append(build_status.getNumber())
599 results['builders'][builderName] = builds
600 return results
601
602
604 help = """List the registered slaves.
605 """
606 title = 'Slaves'
607
609 JsonResource.__init__(self, status)
610 for slave_name in status.getSlaveNames():
611 self.putChild(slave_name,
612 SlaveJsonResource(status,
613 status.getSlave(slave_name)))
614
615
617 help = """Describe the sources for a BuildRequest.
618 """
619 title = 'SourceStamp'
620
622 # buildbot.sourcestamp.SourceStamp
623 JsonResource.__init__(self, status)
624 self.source_stamp = source_stamp
625 self.putChild('changes',
626 ChangesJsonResource(status, source_stamp.changes))
627 # TODO(maruel): Should redirect to the patch's url instead.
628 #if source_stamp.patch:
629 # self.putChild('patch', StaticHTML(source_stamp.path))
630
632 return self.source_stamp.asDict()
633
634
636 """Retrieves all json data."""
637 help = """JSON status
638
639 Root page to give a fair amount of information in the current buildbot master
640 status. You may want to use a child instead to reduce the load on the server.
641
642 For help on any sub directory, use url /child/help
643 """
644 title = 'Buildbot JSON'
645
647 JsonResource.__init__(self, status)
648 self.level = 1
649 self.putChild('builders', BuildersJsonResource(status))
650 self.putChild('change_sources', ChangeSourcesJsonResource(status))
651 self.putChild('project', ProjectJsonResource(status))
652 self.putChild('slaves', SlavesJsonResource(status))
653 # This needs to be called before the first HelpResource().body call.
654 self.hackExamples()
655
657 result = JsonResource.content(self, request)
658 # This is done to hook the downloaded filename.
659 request.path = 'buildbot'
660 return result
661
663 global EXAMPLES
664 # Find the first builder with a previous build or select the last one.
665 builder = None
666 for b in self.status.getBuilderNames():
667 builder = self.status.getBuilder(b)
668 if builder.getBuild(-1):
669 break
670 if not builder:
671 return
672 EXAMPLES = EXAMPLES.replace('<A_BUILDER>', builder.getName())
673 build = builder.getBuild(-1)
674 if build:
675 EXAMPLES = EXAMPLES.replace('<A_BUILD>', str(build.getNumber()))
676 if builder.slavenames:
677 EXAMPLES = EXAMPLES.replace('<A_SLAVE>', builder.slavenames[0])
678
679 # vim: set ts=4 sts=4 sw=4 et:
680
| Home | Trees | Indices | Help |
|
|---|
| Generated by Epydoc 3.0.1 on Tue May 25 17:53:21 2010 | http://epydoc.sourceforge.net |