| Trees | Indices | Help |
|
|---|
|
|
1 # This file is part of Buildbot. Buildbot is free software: you can
2 # redistribute it and/or modify it under the terms of the GNU General Public
3 # License as published by the Free Software Foundation, version 2.
4 #
5 # This program is distributed in the hope that it will be useful, but WITHOUT
6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
8 # details.
9 #
10 # You should have received a copy of the GNU General Public License along with
11 # this program; if not, write to the Free Software Foundation, Inc., 51
12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
13 #
14 # Portions Copyright Buildbot Team Members
15 # Original Copyright (c) 2010 The Chromium Authors.
16
17 """Simple JSON exporter."""
18
19 import datetime
20 import os
21 import re
22
23 from twisted.web import error, html, resource
24
25 from buildbot.status.web.base import HtmlResource
26 from buildbot.util import json
27
28
29 _IS_INT = re.compile('^[-+]?\d+$')
30
31
32 FLAGS = """\
33 - as_text
34 - By default, application/json is used. Setting as_text=1 change the type
35 to text/plain and implicitly sets compact=0 and filter=1. Mainly useful to
36 look at the result in a web browser.
37 - compact
38 - By default, the json data is compact and defaults to 1. For easier to read
39 indented output, set compact=0.
40 - select
41 - By default, most children data is listed. You can do a random selection
42 of data by using select=<sub-url> multiple times to coagulate data.
43 "select=" includes the actual url otherwise it is skipped.
44 - filter
45 - Filters out null, false, and empty string, list and dict. This reduce the
46 amount of useless data sent.
47 - callback
48 - Enable uses of JSONP as described in
49 http://en.wikipedia.org/wiki/JSON#JSONP. Note that
50 Access-Control-Allow-Origin:* is set in the HTTP response header so you
51 can use this in compatible browsers.
52 """
53
54 EXAMPLES = """\
55 - /json
56 - Root node, that *doesn't* mean all the data. Many things (like logs) must
57 be explicitly queried for performance reasons.
58 - /json/builders/
59 - All builders.
60 - /json/builders/<A_BUILDER>
61 - A specific builder as compact text.
62 - /json/builders/<A_BUILDER>/builds
63 - All *cached* builds.
64 - /json/builders/<A_BUILDER>/builds/_all
65 - All builds. Warning, reads all previous build data.
66 - /json/builders/<A_BUILDER>/builds/<A_BUILD>
67 - Where <A_BUILD> is either positive, a build number, or negative, a past
68 build.
69 - /json/builders/<A_BUILDER>/builds/-1/source_stamp/changes
70 - Build changes
71 - /json/builders/<A_BUILDER>/builds?select=-1&select=-2
72 - Two last builds on '<A_BUILDER>' builder.
73 - /json/builders/<A_BUILDER>/builds?select=-1/source_stamp/changes&select=-2/source_stamp/changes
74 - Changes of the two last builds on '<A_BUILDER>' builder.
75 - /json/builders/<A_BUILDER>/slaves
76 - Slaves associated to this builder.
77 - /json/builders/<A_BUILDER>?select=&select=slaves
78 - Builder information plus details information about its slaves. Neat eh?
79 - /json/slaves/<A_SLAVE>
80 - A specific slave.
81 - /json?select=slaves/<A_SLAVE>/&select=project&select=builders/<A_BUILDER>/builds/<A_BUILD>
82 - A selection of random unrelated stuff as an random example. :)
83 """
84
85
87 return request.args.get(arg, [default])[0]
88
89
91 value = RequestArg(request, arg, default)
92 if value in (False, True):
93 return value
94 value = value.lower()
95 if value in ('1', 'true'):
96 return True
97 if value in ('0', 'false'):
98 return False
99 # Ignore value.
100 return default
101
102
104 """Additional method for twisted.web.error.Error."""
105 result = {}
106 result['http_error'] = self.status
107 result['response'] = self.response
108 return result
109
110
112 """Additional method for twisted.web.error.Error."""
113 result = {}
114 result['http_error'] = self.code
115 result['response'] = self.brief
116 result['detail'] = self.detail
117 return result
118
119
120 # Add .asDict() method to twisted.web.error.Error to simplify the code below.
121 error.Error.asDict = TwistedWebErrorAsDict
122 error.PageRedirect.asDict = TwistedWebErrorAsDict
123 error.ErrorPage.asDict = TwistedWebErrorPageAsDict
124 error.NoResource.asDict = TwistedWebErrorPageAsDict
125 error.ForbiddenResource.asDict = TwistedWebErrorPageAsDict
126
127
129 """Returns a copy with None, False, "", [], () and {} removed.
130 Warning: converts tuple to list."""
131 if isinstance(data, (list, tuple)):
132 # Recurse in every items and filter them out.
133 items = map(FilterOut, data)
134 if not filter(lambda x: not x in ('', False, None, [], {}, ()), items):
135 return None
136 return items
137 elif isinstance(data, dict):
138 return dict(filter(lambda x: not x[1] in ('', False, None, [], {}, ()),
139 [(k, FilterOut(v)) for (k, v) in data.iteritems()]))
140 else:
141 return data
142
143
145 """Base class for json data."""
146
147 contentType = "application/json"
148 cache_seconds = 60
149 help = None
150 title = None
151 level = 0
152
154 """Adds transparent lazy-child initialization."""
155 resource.Resource.__init__(self)
156 # buildbot.status.builder.Status
157 self.status = status
158 if self.help:
159 title = ''
160 if self.title:
161 title = self.title + ' help'
162 self.putChild('help',
163 HelpResource(self.help, title=title, parent_node=self))
164
166 """Adds transparent support for url ending with /"""
167 if path == "" and len(request.postpath) == 0:
168 return self
169 # Equivalent to resource.Resource.getChildWithDefault()
170 if self.children.has_key(path):
171 return self.children[path]
172 return self.getChild(path, request)
173
175 """Adds the resource's level for help links generation."""
176
177 def RecurseFix(res, level):
178 res.level = level + 1
179 for c in res.children.itervalues():
180 RecurseFix(c, res.level)
181
182 RecurseFix(res, self.level)
183 resource.Resource.putChild(self, name, res)
184
186 """Renders a HTTP GET at the http request level."""
187 data = self.content(request)
188 if isinstance(data, unicode):
189 data = data.encode("utf-8")
190 request.setHeader("Access-Control-Allow-Origin", "*")
191 if RequestArgToBool(request, 'as_text', False):
192 request.setHeader("content-type", 'text/plain')
193 else:
194 request.setHeader("content-type", self.contentType)
195 request.setHeader("content-disposition",
196 "attachment; filename=\"%s.json\"" % request.path)
197 # Make sure we get fresh pages.
198 if self.cache_seconds:
199 now = datetime.datetime.utcnow()
200 expires = now + datetime.timedelta(seconds=self.cache_seconds)
201 request.setHeader("Expires",
202 expires.strftime("%a, %d %b %Y %H:%M:%S GMT"))
203 request.setHeader("Pragma", "no-cache")
204 return data
205
207 """Renders the json dictionaries."""
208 # Supported flags.
209 select = request.args.get('select')
210 as_text = RequestArgToBool(request, 'as_text', False)
211 filter_out = RequestArgToBool(request, 'filter', as_text)
212 compact = RequestArgToBool(request, 'compact', not as_text)
213 callback = request.args.get('callback')
214
215 # Implement filtering at global level and every child.
216 if select is not None:
217 del request.args['select']
218 # Do not render self.asDict()!
219 data = {}
220 # Remove superfluous /
221 select = [s.strip('/') for s in select]
222 select.sort(cmp=lambda x,y: cmp(x.count('/'), y.count('/')),
223 reverse=True)
224 for item in select:
225 # Start back at root.
226 node = data
227 # Implementation similar to twisted.web.resource.getChildForRequest
228 # but with a hacked up request.
229 child = self
230 prepath = request.prepath[:]
231 postpath = request.postpath[:]
232 request.postpath = filter(None, item.split('/'))
233 while request.postpath and not child.isLeaf:
234 pathElement = request.postpath.pop(0)
235 node[pathElement] = {}
236 node = node[pathElement]
237 request.prepath.append(pathElement)
238 child = child.getChildWithDefault(pathElement, request)
239 node.update(child.asDict(request))
240 request.prepath = prepath
241 request.postpath = postpath
242 else:
243 data = self.asDict(request)
244 if filter_out:
245 data = FilterOut(data)
246 if compact:
247 data = json.dumps(data, sort_keys=True, separators=(',',':'))
248 else:
249 data = json.dumps(data, sort_keys=True, indent=2)
250 if callback:
251 # Only accept things that look like identifiers for now
252 callback = callback[0]
253 if re.match(r'^[a-zA-Z$][a-zA-Z$0-9.]*$', callback):
254 data = '%s(%s);' % (callback, data)
255 return data
256
258 """Generates the json dictionary.
259
260 By default, renders every childs."""
261 if self.children:
262 data = {}
263 for name in self.children:
264 child = self.getChildWithDefault(name, request)
265 if isinstance(child, JsonResource):
266 data[name] = child.asDict(request)
267 # else silently pass over non-json resources.
268 return data
269 else:
270 raise NotImplementedError()
271
272
274 """Convert a string in a wiki-style format into HTML."""
275 indent = 0
276 in_item = False
277 output = []
278 for line in text.splitlines(False):
279 match = re.match(r'^( +)\- (.*)$', line)
280 if match:
281 if indent < len(match.group(1)):
282 output.append('<ul>')
283 indent = len(match.group(1))
284 elif indent > len(match.group(1)):
285 while indent > len(match.group(1)):
286 output.append('</ul>')
287 indent -= 2
288 if in_item:
289 # Close previous item
290 output.append('</li>')
291 output.append('<li>')
292 in_item = True
293 line = match.group(2)
294 elif indent:
295 if line.startswith((' ' * indent) + ' '):
296 # List continuation
297 line = line.strip()
298 else:
299 # List is done
300 if in_item:
301 output.append('</li>')
302 in_item = False
303 while indent > 0:
304 output.append('</ul>')
305 indent -= 2
306
307 if line.startswith('/'):
308 if not '?' in line:
309 line_full = line + '?as_text=1'
310 else:
311 line_full = line + '&as_text=1'
312 output.append('<a href="' + html.escape(line_full) + '">' +
313 html.escape(line) + '</a>')
314 else:
315 output.append(html.escape(line).replace(' ', ' '))
316 if not in_item:
317 output.append('<br>')
318
319 if in_item:
320 output.append('</li>')
321 while indent > 0:
322 output.append('</ul>')
323 indent -= 2
324 return '\n'.join(output)
325
326
329 HtmlResource.__init__(self)
330 self.text = text
331 self.title = title
332 self.parent_node = parent_node
333
335 cxt['level'] = self.parent_node.level
336 cxt['text'] = ToHtml(self.text)
337 cxt['children'] = [ n for n in self.parent_node.children.keys() if n != 'help' ]
338 cxt['flags'] = ToHtml(FLAGS)
339 cxt['examples'] = ToHtml(EXAMPLES).replace(
340 'href="/json',
341 'href="%sjson' % (self.level * '../'))
342
343 template = request.site.buildbot_service.templates.get_template("jsonhelp.html")
344 return template.render(**cxt)
345
347 help = """Describe a single builder.
348 """
349 title = 'Builder'
350
352 JsonResource.__init__(self, status)
353 self.builder_status = builder_status
354 self.putChild('builds', BuildsJsonResource(status, builder_status))
355 self.putChild('slaves', BuilderSlavesJsonResources(status,
356 builder_status))
357
361
362
364 help = """List of all the builders defined on a master.
365 """
366 title = 'Builders'
367
369 JsonResource.__init__(self, status)
370 for builder_name in self.status.getBuilderNames():
371 self.putChild(builder_name,
372 BuilderJsonResource(status,
373 status.getBuilder(builder_name)))
374
375
377 help = """Describe the slaves attached to a single builder.
378 """
379 title = 'BuilderSlaves'
380
382 JsonResource.__init__(self, status)
383 self.builder_status = builder_status
384 for slave_name in self.builder_status.slavenames:
385 self.putChild(slave_name,
386 SlaveJsonResource(status,
387 self.status.getSlave(slave_name)))
388
389
391 help = """Describe a single build.
392 """
393 title = 'Build'
394
396 JsonResource.__init__(self, status)
397 self.build_status = build_status
398 self.putChild('source_stamp',
399 SourceStampJsonResource(status,
400 build_status.getSourceStamp()))
401 self.putChild('steps', BuildStepsJsonResource(status, build_status))
402
405
406
408 help = """All the builds that were run on a builder.
409 """
410 title = 'AllBuilds'
411
415
417 # Dynamic childs.
418 if isinstance(path, int) or _IS_INT.match(path):
419 build_status = self.builder_status.getBuild(int(path))
420 if build_status:
421 build_status_number = str(build_status.getNumber())
422 # Happens with negative numbers.
423 child = self.children.get(build_status_number)
424 if child:
425 return child
426 # Create it on-demand.
427 child = BuildJsonResource(self.status, build_status)
428 # Cache it. Never cache negative numbers.
429 # TODO(maruel): Cleanup the cache once it's too heavy!
430 self.putChild(build_status_number, child)
431 return child
432 return JsonResource.getChild(self, path, request)
433
435 results = {}
436 # If max > buildCacheSize, it'll trash the cache...
437 max = int(RequestArg(request, 'max',
438 self.builder_status.buildCacheSize))
439 for i in range(0, max):
440 child = self.getChildWithDefault(-i, request)
441 if not isinstance(child, BuildJsonResource):
442 continue
443 results[child.build_status.getNumber()] = child.asDict(request)
444 return results
445
446
448 help = """Builds that were run on a builder.
449 """
450 title = 'Builds'
451
453 AllBuildsJsonResource.__init__(self, status, builder_status)
454 self.putChild('_all', AllBuildsJsonResource(status, builder_status))
455
457 # Transparently redirects to _all if path is not ''.
458 return self.children['_all'].getChildWithDefault(path, request)
459
461 # This would load all the pickles and is way too heavy, especially that
462 # it would trash the cache:
463 # self.children['builds'].asDict(request)
464 # TODO(maruel) This list should also need to be cached but how?
465 builds = dict([
466 (int(file), None)
467 for file in os.listdir(self.builder_status.basedir)
468 if _IS_INT.match(file)
469 ])
470 return builds
471
472
474 help = """A single build step.
475 """
476 title = 'BuildStep'
477
479 # buildbot.status.builder.BuildStepStatus
480 JsonResource.__init__(self, status)
481 self.build_step_status = build_step_status
482 # TODO self.putChild('logs', LogsJsonResource())
483
485 return self.build_step_status.asDict()
486
487
489 help = """A list of build steps that occurred during a build.
490 """
491 title = 'BuildSteps'
492
496 # The build steps are constantly changing until the build is done so
497 # keep a reference to build_status instead
498
500 # Dynamic childs.
501 build_step_status = None
502 if isinstance(path, int) or _IS_INT.match(path):
503 build_step_status = self.build_status.getSteps[int(path)]
504 else:
505 steps_dict = dict([(step.getName(), step)
506 for step in self.build_status.getStep()])
507 build_step_status = steps_dict.get(path)
508 if build_step_status:
509 # Create it on-demand.
510 child = BuildStepJsonResource(self.status, build_step_status)
511 # Cache it.
512 index = self.build_status.getSteps().index(build_step_status)
513 self.putChild(str(index), child)
514 self.putChild(build_step_status.getName(), child)
515 return child
516 return JsonResource.getChild(self, path, request)
517
526
527
529 help = """Describe a single change that originates from a change source.
530 """
531 title = 'Change'
532
534 # buildbot.changes.changes.Change
535 JsonResource.__init__(self, status)
536 self.change = change
537
539 return self.change.asDict()
540
541
543 help = """List of changes.
544 """
545 title = 'Changes'
546
548 JsonResource.__init__(self, status)
549 for c in changes:
550 # TODO(maruel): Problem with multiple changes with the same number.
551 # Probably try server hack specific so we could fix it on this side
552 # instead. But there is still the problem with multiple pollers from
553 # different repo where the numbers could clash.
554 number = str(c.number)
555 while number in self.children:
556 # TODO(maruel): Do something better?
557 number = str(int(c.number)+1)
558 self.putChild(number, ChangeJsonResource(status, c))
559
561 """Don't throw an exception when there is no child."""
562 if not self.children:
563 return {}
564 return JsonResource.asDict(self, request)
565
566
568 help = """Describe a change source.
569 """
570 title = 'ChangeSources'
571
573 result = {}
574 n = 0
575 for c in self.status.getChangeSources():
576 # buildbot.changes.changes.ChangeMaster
577 change = {}
578 change['description'] = c.describe()
579 result[n] = change
580 n += 1
581 return result
582
583
591
592
594 help = """Describe a slave.
595 """
596 title = 'Slave'
597
599 JsonResource.__init__(self, status)
600 self.slave_status = slave_status
601 self.name = self.slave_status.getName()
602 self.builders = None
603
605 if self.builders is None:
606 # Figure out all the builders to which it's attached
607 self.builders = []
608 for builderName in self.status.getBuilderNames():
609 if self.name in self.status.getBuilder(builderName).slavenames:
610 self.builders.append(builderName)
611 return self.builders
612
614 results = self.slave_status.asDict()
615 # Enhance it by adding more informations.
616 results['builders'] = {}
617 for builderName in self.getBuilders():
618 builds = []
619 builder_status = self.status.getBuilder(builderName)
620 for i in range(1, builder_status.buildCacheSize - 1):
621 build_status = builder_status.getBuild(-i)
622 if not build_status or not build_status.isFinished():
623 # If not finished, it will appear in runningBuilds.
624 break
625 if build_status.getSlavename() == self.name:
626 builds.append(build_status.getNumber())
627 results['builders'][builderName] = builds
628 return results
629
630
632 help = """List the registered slaves.
633 """
634 title = 'Slaves'
635
637 JsonResource.__init__(self, status)
638 for slave_name in status.getSlaveNames():
639 self.putChild(slave_name,
640 SlaveJsonResource(status,
641 status.getSlave(slave_name)))
642
643
645 help = """Describe the sources for a BuildRequest.
646 """
647 title = 'SourceStamp'
648
650 # buildbot.sourcestamp.SourceStamp
651 JsonResource.__init__(self, status)
652 self.source_stamp = source_stamp
653 self.putChild('changes',
654 ChangesJsonResource(status, source_stamp.changes))
655 # TODO(maruel): Should redirect to the patch's url instead.
656 #if source_stamp.patch:
657 # self.putChild('patch', StaticHTML(source_stamp.path))
658
660 return self.source_stamp.asDict()
661
662
664 """Retrieves all json data."""
665 help = """JSON status
666
667 Root page to give a fair amount of information in the current buildbot master
668 status. You may want to use a child instead to reduce the load on the server.
669
670 For help on any sub directory, use url /child/help
671 """
672 title = 'Buildbot JSON'
673
675 JsonResource.__init__(self, status)
676 self.level = 1
677 self.putChild('builders', BuildersJsonResource(status))
678 self.putChild('change_sources', ChangeSourcesJsonResource(status))
679 self.putChild('project', ProjectJsonResource(status))
680 self.putChild('slaves', SlavesJsonResource(status))
681 # This needs to be called before the first HelpResource().body call.
682 self.hackExamples()
683
685 result = JsonResource.content(self, request)
686 # This is done to hook the downloaded filename.
687 request.path = 'buildbot'
688 return result
689
691 global EXAMPLES
692 # Find the first builder with a previous build or select the last one.
693 builder = None
694 for b in self.status.getBuilderNames():
695 builder = self.status.getBuilder(b)
696 if builder.getBuild(-1):
697 break
698 if not builder:
699 return
700 EXAMPLES = EXAMPLES.replace('<A_BUILDER>', builder.getName())
701 build = builder.getBuild(-1)
702 if build:
703 EXAMPLES = EXAMPLES.replace('<A_BUILD>', str(build.getNumber()))
704 if builder.slavenames:
705 EXAMPLES = EXAMPLES.replace('<A_SLAVE>', builder.slavenames[0])
706
707 # vim: set ts=4 sts=4 sw=4 et:
708
| Trees | Indices | Help |
|
|---|
| Generated by Epydoc 3.0.1 on Sun Dec 19 18:26:49 2010 | http://epydoc.sourceforge.net |