| Trees | Indices | Help |
|
|---|
|
|
1 # -*- test-case-name: buildbot.test.test_web_status_json -*-
2 # Original Copyright (c) 2010 The Chromium Authors.
3
4 """Simple JSON exporter."""
5
6 import datetime
7 import os
8 import re
9
10 from twisted.web import error, html, resource
11
12 from buildbot.status.web.base import HtmlResource
13 from buildbot.util import json
14
15
16 _IS_INT = re.compile('^[-+]?\d+$')
17
18
19 FLAGS = """\
20 - as_text
21 - By default, application/json is used. Setting as_text=1 change the type
22 to text/plain and implicitly sets compact=0 and filter=1. Mainly useful to
23 look at the result in a web browser.
24 - compact
25 - By default, the json data is compact and defaults to 1. For easier to read
26 indented output, set compact=0.
27 - select
28 - By default, most children data is listed. You can do a random selection
29 of data by using select=<sub-url> multiple times to coagulate data.
30 "select=" includes the actual url otherwise it is skipped.
31 - filter
32 - Filters out null, false, and empty string, list and dict. This reduce the
33 amount of useless data sent.
34 - callback
35 - Enable uses of JSONP as described in
36 http://en.wikipedia.org/wiki/JSON#JSONP. Note that
37 Access-Control-Allow-Origin:* is set in the HTTP response header so you
38 can use this in compatible browsers.
39 """
40
41 EXAMPLES = """\
42 - /json
43 - Root node, that *doesn't* mean all the data. Many things (like logs) must
44 be explicitly queried for performance reasons.
45 - /json/builders/
46 - All builders.
47 - /json/builders/<A_BUILDER>
48 - A specific builder as compact text.
49 - /json/builders/<A_BUILDER>/builds
50 - All *cached* builds.
51 - /json/builders/<A_BUILDER>/builds/_all
52 - All builds. Warning, reads all previous build data.
53 - /json/builders/<A_BUILDER>/builds/<A_BUILD>
54 - Where <A_BUILD> is either positive, a build number, or negative, a past
55 build.
56 - /json/builders/<A_BUILDER>/builds/-1/source_stamp/changes
57 - Build changes
58 - /json/builders/<A_BUILDER>/builds?select=-1&select=-2
59 - Two last builds on '<A_BUILDER>' builder.
60 - /json/builders/<A_BUILDER>/builds?select=-1/source_stamp/changes&select=-2/source_stamp/changes
61 - Changes of the two last builds on '<A_BUILDER>' builder.
62 - /json/builders/<A_BUILDER>/slaves
63 - Slaves associated to this builder.
64 - /json/builders/<A_BUILDER>?select=&select=slaves
65 - Builder information plus details information about its slaves. Neat eh?
66 - /json/slaves/<A_SLAVE>
67 - A specific slave.
68 - /json?select=slaves/<A_SLAVE>/&select=project&select=builders/<A_BUILDER>/builds/<A_BUILD>
69 - A selection of random unrelated stuff as an random example. :)
70 """
71
72
74 return request.args.get(arg, [default])[0]
75
76
78 value = RequestArg(request, arg, default)
79 if value in (False, True):
80 return value
81 value = value.lower()
82 if value in ('1', 'true'):
83 return True
84 if value in ('0', 'false'):
85 return False
86 # Ignore value.
87 return default
88
89
91 """Additional method for twisted.web.error.Error."""
92 result = {}
93 result['http_error'] = self.status
94 result['response'] = self.response
95 return result
96
97
99 """Additional method for twisted.web.error.Error."""
100 result = {}
101 result['http_error'] = self.code
102 result['response'] = self.brief
103 result['detail'] = self.detail
104 return result
105
106
107 # Add .asDict() method to twisted.web.error.Error to simplify the code below.
108 error.Error.asDict = TwistedWebErrorAsDict
109 error.PageRedirect.asDict = TwistedWebErrorAsDict
110 error.ErrorPage.asDict = TwistedWebErrorPageAsDict
111 error.NoResource.asDict = TwistedWebErrorPageAsDict
112 error.ForbiddenResource.asDict = TwistedWebErrorPageAsDict
113
114
116 """Returns a copy with None, False, "", [], () and {} removed.
117 Warning: converts tuple to list."""
118 if isinstance(data, (list, tuple)):
119 # Recurse in every items and filter them out.
120 items = map(FilterOut, data)
121 if not filter(lambda x: not x in ('', False, None, [], {}, ()), items):
122 return None
123 return items
124 elif isinstance(data, dict):
125 return dict(filter(lambda x: not x[1] in ('', False, None, [], {}, ()),
126 [(k, FilterOut(v)) for (k, v) in data.iteritems()]))
127 else:
128 return data
129
130
132 """Base class for json data."""
133
134 contentType = "application/json"
135 cache_seconds = 60
136 help = None
137 title = None
138 level = 0
139
141 """Adds transparent lazy-child initialization."""
142 resource.Resource.__init__(self)
143 # buildbot.status.builder.Status
144 self.status = status
145 if self.help:
146 title = ''
147 if self.title:
148 title = self.title + ' help'
149 self.putChild('help',
150 HelpResource(self.help, title=title, parent_node=self))
151
153 """Adds transparent support for url ending with /"""
154 if path == "" and len(request.postpath) == 0:
155 return self
156 # Equivalent to resource.Resource.getChildWithDefault()
157 if self.children.has_key(path):
158 return self.children[path]
159 return self.getChild(path, request)
160
162 """Adds the resource's level for help links generation."""
163
164 def RecurseFix(res, level):
165 res.level = level + 1
166 for c in res.children.itervalues():
167 RecurseFix(c, res.level)
168
169 RecurseFix(res, self.level)
170 resource.Resource.putChild(self, name, res)
171
173 """Renders a HTTP GET at the http request level."""
174 data = self.content(request)
175 if isinstance(data, unicode):
176 data = data.encode("utf-8")
177 request.setHeader("Access-Control-Allow-Origin", "*")
178 if RequestArgToBool(request, 'as_text', False):
179 request.setHeader("content-type", 'text/plain')
180 else:
181 request.setHeader("content-type", self.contentType)
182 request.setHeader("content-disposition",
183 "attachment; filename=\"%s.json\"" % request.path)
184 # Make sure we get fresh pages.
185 if self.cache_seconds:
186 now = datetime.datetime.utcnow()
187 expires = now + datetime.timedelta(seconds=self.cache_seconds)
188 request.setHeader("Expires",
189 expires.strftime("%a, %d %b %Y %H:%M:%S GMT"))
190 request.setHeader("Pragma", "no-cache")
191 return data
192
194 """Renders the json dictionaries."""
195 # Supported flags.
196 select = request.args.get('select')
197 as_text = RequestArgToBool(request, 'as_text', False)
198 filter_out = RequestArgToBool(request, 'filter', as_text)
199 compact = RequestArgToBool(request, 'compact', not as_text)
200 callback = request.args.get('callback')
201
202 # Implement filtering at global level and every child.
203 if select is not None:
204 del request.args['select']
205 # Do not render self.asDict()!
206 data = {}
207 # Remove superfluous /
208 select = [s.strip('/') for s in select]
209 select.sort(cmp=lambda x,y: cmp(x.count('/'), y.count('/')),
210 reverse=True)
211 for item in select:
212 # Start back at root.
213 node = data
214 # Implementation similar to twisted.web.resource.getChildForRequest
215 # but with a hacked up request.
216 child = self
217 prepath = request.prepath[:]
218 postpath = request.postpath[:]
219 request.postpath = filter(None, item.split('/'))
220 while request.postpath and not child.isLeaf:
221 pathElement = request.postpath.pop(0)
222 node[pathElement] = {}
223 node = node[pathElement]
224 request.prepath.append(pathElement)
225 child = child.getChildWithDefault(pathElement, request)
226 node.update(child.asDict(request))
227 request.prepath = prepath
228 request.postpath = postpath
229 else:
230 data = self.asDict(request)
231 if filter_out:
232 data = FilterOut(data)
233 if compact:
234 data = json.dumps(data, sort_keys=True, separators=(',',':'))
235 else:
236 data = json.dumps(data, sort_keys=True, indent=2)
237 if callback:
238 # Only accept things that look like identifiers for now
239 callback = callback[0]
240 if re.match(r'^[a-zA-Z$][a-zA-Z$0-9.]*$', callback):
241 data = '%s(%s);' % (callback, data)
242 return data
243
245 """Generates the json dictionary.
246
247 By default, renders every childs."""
248 if self.children:
249 data = {}
250 for name in self.children:
251 child = self.getChildWithDefault(name, request)
252 if isinstance(child, JsonResource):
253 data[name] = child.asDict(request)
254 # else silently pass over non-json resources.
255 return data
256 else:
257 raise NotImplementedError()
258
259
261 """Convert a string in a wiki-style format into HTML."""
262 indent = 0
263 in_item = False
264 output = []
265 for line in text.splitlines(False):
266 match = re.match(r'^( +)\- (.*)$', line)
267 if match:
268 if indent < len(match.group(1)):
269 output.append('<ul>')
270 indent = len(match.group(1))
271 elif indent > len(match.group(1)):
272 while indent > len(match.group(1)):
273 output.append('</ul>')
274 indent -= 2
275 if in_item:
276 # Close previous item
277 output.append('</li>')
278 output.append('<li>')
279 in_item = True
280 line = match.group(2)
281 elif indent:
282 if line.startswith((' ' * indent) + ' '):
283 # List continuation
284 line = line.strip()
285 else:
286 # List is done
287 if in_item:
288 output.append('</li>')
289 in_item = False
290 while indent > 0:
291 output.append('</ul>')
292 indent -= 2
293
294 if line.startswith('/'):
295 if not '?' in line:
296 line_full = line + '?as_text=1'
297 else:
298 line_full = line + '&as_text=1'
299 output.append('<a href="' + html.escape(line_full) + '">' +
300 html.escape(line) + '</a>')
301 else:
302 output.append(html.escape(line).replace(' ', ' '))
303 if not in_item:
304 output.append('<br>')
305
306 if in_item:
307 output.append('</li>')
308 while indent > 0:
309 output.append('</ul>')
310 indent -= 2
311 return '\n'.join(output)
312
313
316 HtmlResource.__init__(self)
317 self.text = text
318 self.title = title
319 self.parent_node = parent_node
320
322 cxt['level'] = self.parent_node.level
323 cxt['text'] = ToHtml(self.text)
324 cxt['children'] = [ n for n in self.parent_node.children.keys() if n != 'help' ]
325 cxt['flags'] = ToHtml(FLAGS)
326 cxt['examples'] = ToHtml(EXAMPLES).replace(
327 'href="/json',
328 'href="%sjson' % (self.level * '../'))
329
330 template = request.site.buildbot_service.templates.get_template("jsonhelp.html")
331 return template.render(**cxt)
332
334 help = """Describe a single builder.
335 """
336 title = 'Builder'
337
339 JsonResource.__init__(self, status)
340 self.builder_status = builder_status
341 self.putChild('builds', BuildsJsonResource(status, builder_status))
342 self.putChild('slaves', BuilderSlavesJsonResources(status,
343 builder_status))
344
348
349
351 help = """List of all the builders defined on a master.
352 """
353 title = 'Builders'
354
356 JsonResource.__init__(self, status)
357 for builder_name in self.status.getBuilderNames():
358 self.putChild(builder_name,
359 BuilderJsonResource(status,
360 status.getBuilder(builder_name)))
361
362
364 help = """Describe the slaves attached to a single builder.
365 """
366 title = 'BuilderSlaves'
367
369 JsonResource.__init__(self, status)
370 self.builder_status = builder_status
371 for slave_name in self.builder_status.slavenames:
372 self.putChild(slave_name,
373 SlaveJsonResource(status,
374 self.status.getSlave(slave_name)))
375
376
378 help = """Describe a single build.
379 """
380 title = 'Build'
381
383 JsonResource.__init__(self, status)
384 self.build_status = build_status
385 self.putChild('source_stamp',
386 SourceStampJsonResource(status,
387 build_status.getSourceStamp()))
388 self.putChild('steps', BuildStepsJsonResource(status, build_status))
389
392
393
395 help = """All the builds that were run on a builder.
396 """
397 title = 'AllBuilds'
398
402
404 # Dynamic childs.
405 if isinstance(path, int) or _IS_INT.match(path):
406 build_status = self.builder_status.getBuild(int(path))
407 if build_status:
408 build_status_number = str(build_status.getNumber())
409 # Happens with negative numbers.
410 child = self.children.get(build_status_number)
411 if child:
412 return child
413 # Create it on-demand.
414 child = BuildJsonResource(self.status, build_status)
415 # Cache it. Never cache negative numbers.
416 # TODO(maruel): Cleanup the cache once it's too heavy!
417 self.putChild(build_status_number, child)
418 return child
419 return JsonResource.getChild(self, path, request)
420
422 results = {}
423 # If max > buildCacheSize, it'll trash the cache...
424 max = int(RequestArg(request, 'max',
425 self.builder_status.buildCacheSize))
426 for i in range(0, max):
427 child = self.getChildWithDefault(-i, request)
428 if not isinstance(child, BuildJsonResource):
429 continue
430 results[child.build_status.getNumber()] = child.asDict(request)
431 return results
432
433
435 help = """Builds that were run on a builder.
436 """
437 title = 'Builds'
438
440 AllBuildsJsonResource.__init__(self, status, builder_status)
441 self.putChild('_all', AllBuildsJsonResource(status, builder_status))
442
444 # Transparently redirects to _all if path is not ''.
445 return self.children['_all'].getChildWithDefault(path, request)
446
448 # This would load all the pickles and is way too heavy, especially that
449 # it would trash the cache:
450 # self.children['builds'].asDict(request)
451 # TODO(maruel) This list should also need to be cached but how?
452 builds = dict([
453 (int(file), None)
454 for file in os.listdir(self.builder_status.basedir)
455 if _IS_INT.match(file)
456 ])
457 return builds
458
459
461 help = """A single build step.
462 """
463 title = 'BuildStep'
464
466 # buildbot.status.builder.BuildStepStatus
467 JsonResource.__init__(self, status)
468 self.build_step_status = build_step_status
469 # TODO self.putChild('logs', LogsJsonResource())
470
472 return self.build_step_status.asDict()
473
474
476 help = """A list of build steps that occurred during a build.
477 """
478 title = 'BuildSteps'
479
483 # The build steps are constantly changing until the build is done so
484 # keep a reference to build_status instead
485
487 # Dynamic childs.
488 build_step_status = None
489 if isinstance(path, int) or _IS_INT.match(path):
490 build_step_status = self.build_status.getSteps[int(path)]
491 else:
492 steps_dict = dict([(step.getName(), step)
493 for step in self.build_status.getStep()])
494 build_step_status = steps_dict.get(path)
495 if build_step_status:
496 # Create it on-demand.
497 child = BuildStepJsonResource(self.status, build_step_status)
498 # Cache it.
499 index = self.build_status.getSteps().index(build_step_status)
500 self.putChild(str(index), child)
501 self.putChild(build_step_status.getName(), child)
502 return child
503 return JsonResource.getChild(self, path, request)
504
513
514
516 help = """Describe a single change that originates from a change source.
517 """
518 title = 'Change'
519
521 # buildbot.changes.changes.Change
522 JsonResource.__init__(self, status)
523 self.change = change
524
526 return self.change.asDict()
527
528
530 help = """List of changes.
531 """
532 title = 'Changes'
533
535 JsonResource.__init__(self, status)
536 for c in changes:
537 # TODO(maruel): Problem with multiple changes with the same number.
538 # Probably try server hack specific so we could fix it on this side
539 # instead. But there is still the problem with multiple pollers from
540 # different repo where the numbers could clash.
541 number = str(c.number)
542 while number in self.children:
543 # TODO(maruel): Do something better?
544 number = str(int(c.number)+1)
545 self.putChild(number, ChangeJsonResource(status, c))
546
548 """Don't throw an exception when there is no child."""
549 if not self.children:
550 return {}
551 return JsonResource.asDict(self, request)
552
553
555 help = """Describe a change source.
556 """
557 title = 'ChangeSources'
558
560 result = {}
561 n = 0
562 for c in self.status.getChangeSources():
563 # buildbot.changes.changes.ChangeMaster
564 change = {}
565 change['description'] = c.describe()
566 result[n] = change
567 n += 1
568 return result
569
570
578
579
581 help = """Describe a slave.
582 """
583 title = 'Slave'
584
586 JsonResource.__init__(self, status)
587 self.slave_status = slave_status
588 self.name = self.slave_status.getName()
589 self.builders = None
590
592 if self.builders is None:
593 # Figure out all the builders to which it's attached
594 self.builders = []
595 for builderName in self.status.getBuilderNames():
596 if self.name in self.status.getBuilder(builderName).slavenames:
597 self.builders.append(builderName)
598 return self.builders
599
601 results = self.slave_status.asDict()
602 # Enhance it by adding more informations.
603 results['builders'] = {}
604 for builderName in self.getBuilders():
605 builds = []
606 builder_status = self.status.getBuilder(builderName)
607 for i in range(1, builder_status.buildCacheSize - 1):
608 build_status = builder_status.getBuild(-i)
609 if not build_status or not build_status.isFinished():
610 # If not finished, it will appear in runningBuilds.
611 break
612 if build_status.getSlavename() == self.name:
613 builds.append(build_status.getNumber())
614 results['builders'][builderName] = builds
615 return results
616
617
619 help = """List the registered slaves.
620 """
621 title = 'Slaves'
622
624 JsonResource.__init__(self, status)
625 for slave_name in status.getSlaveNames():
626 self.putChild(slave_name,
627 SlaveJsonResource(status,
628 status.getSlave(slave_name)))
629
630
632 help = """Describe the sources for a BuildRequest.
633 """
634 title = 'SourceStamp'
635
637 # buildbot.sourcestamp.SourceStamp
638 JsonResource.__init__(self, status)
639 self.source_stamp = source_stamp
640 self.putChild('changes',
641 ChangesJsonResource(status, source_stamp.changes))
642 # TODO(maruel): Should redirect to the patch's url instead.
643 #if source_stamp.patch:
644 # self.putChild('patch', StaticHTML(source_stamp.path))
645
647 return self.source_stamp.asDict()
648
649
651 """Retrieves all json data."""
652 help = """JSON status
653
654 Root page to give a fair amount of information in the current buildbot master
655 status. You may want to use a child instead to reduce the load on the server.
656
657 For help on any sub directory, use url /child/help
658 """
659 title = 'Buildbot JSON'
660
662 JsonResource.__init__(self, status)
663 self.level = 1
664 self.putChild('builders', BuildersJsonResource(status))
665 self.putChild('change_sources', ChangeSourcesJsonResource(status))
666 self.putChild('project', ProjectJsonResource(status))
667 self.putChild('slaves', SlavesJsonResource(status))
668 # This needs to be called before the first HelpResource().body call.
669 self.hackExamples()
670
672 result = JsonResource.content(self, request)
673 # This is done to hook the downloaded filename.
674 request.path = 'buildbot'
675 return result
676
678 global EXAMPLES
679 # Find the first builder with a previous build or select the last one.
680 builder = None
681 for b in self.status.getBuilderNames():
682 builder = self.status.getBuilder(b)
683 if builder.getBuild(-1):
684 break
685 if not builder:
686 return
687 EXAMPLES = EXAMPLES.replace('<A_BUILDER>', builder.getName())
688 build = builder.getBuild(-1)
689 if build:
690 EXAMPLES = EXAMPLES.replace('<A_BUILD>', str(build.getNumber()))
691 if builder.slavenames:
692 EXAMPLES = EXAMPLES.replace('<A_SLAVE>', builder.slavenames[0])
693
694 # vim: set ts=4 sts=4 sw=4 et:
695
| Trees | Indices | Help |
|
|---|
| Generated by Epydoc 3.0.1 on Fri Oct 29 10:00:50 2010 | http://epydoc.sourceforge.net |