Package buildbot :: Package status :: Package web :: Module status_json
[frames] | no frames]

Source Code for Module buildbot.status.web.status_json

  1  # -*- test-case-name: buildbot.test.test_web_status_json -*- 
  2  # Original Copyright (c) 2010 The Chromium Authors. 
  3   
  4  """Simple JSON exporter.""" 
  5   
  6  import datetime 
  7  import os 
  8  import re 
  9   
 10  from twisted.web import error, html, resource 
 11   
 12  from buildbot.status.web.base import HtmlResource 
 13  from buildbot.util import json 
 14   
 15   
 16  _IS_INT = re.compile('^[-+]?\d+$') 
 17   
 18   
 19  FLAGS = """\ 
 20    - as_text 
 21      - By default, application/json is used. Setting as_text=1 change the type 
 22        to text/plain and implicitly sets compact=0 and filter=1. Mainly useful to 
 23        look at the result in a web browser. 
 24    - compact 
 25      - By default, the json data is compact and defaults to 1. For easier to read 
 26        indented output, set compact=0. 
 27    - select 
 28      - By default, most children data is listed. You can do a random selection 
 29        of data by using select=<sub-url> multiple times to coagulate data. 
 30        "select=" includes the actual url otherwise it is skipped. 
 31    - filter 
 32      - Filters out null, false, and empty string, list and dict. This reduce the 
 33        amount of useless data sent. 
 34   
 35  """ 
 36   
 37  EXAMPLES = """\ 
 38    - /json 
 39      - Root node, that *doesn't* mean all the data. Many things (like logs) must 
 40        be explicitly queried for performance reasons. 
 41    - /json/builders/ 
 42      - All builders. 
 43    - /json/builders/<A_BUILDER> 
 44      - A specific builder as compact text. 
 45    - /json/builders/<A_BUILDER>/builds 
 46      - All *cached* builds. 
 47    - /json/builders/<A_BUILDER>/builds/_all 
 48      - All builds. Warning, reads all previous build data. 
 49    - /json/builders/<A_BUILDER>/builds/<A_BUILD> 
 50      - Where <A_BUILD> is either positive, a build number, or negative, a past 
 51        build. 
 52    - /json/builders/<A_BUILDER>/builds/-1/source_stamp/changes 
 53      - Build changes 
 54    - /json/builders/<A_BUILDER>/builds?select=-1&select=-2 
 55      - Two last builds on '<A_BUILDER>' builder. 
 56    - /json/builders/<A_BUILDER>/builds?select=-1/source_stamp/changes&select=-2/source_stamp/changes 
 57      - Changes of the two last builds on '<A_BUILDER>' builder. 
 58    - /json/builders/<A_BUILDER>/slaves 
 59      - Slaves associated to this builder. 
 60    - /json/builders/<A_BUILDER>?select=&select=slaves 
 61      - Builder information plus details information about its slaves. Neat eh? 
 62    - /json/slaves/<A_SLAVE> 
 63      - A specific slave. 
 64    - /json?select=slaves/<A_SLAVE>/&select=project&select=builders/<A_BUILDER>/builds/<A_BUILD> 
 65      - A selection of random unrelated stuff as an random example. :) 
 66  """ 
 67   
 68   
69 -def RequestArg(request, arg, default):
70 return request.args.get(arg, [default])[0]
71 72
73 -def RequestArgToBool(request, arg, default):
74 value = RequestArg(request, arg, default) 75 if value in (False, True): 76 return value 77 value = value.lower() 78 if value in ('1', 'true'): 79 return True 80 if value in ('0', 'false'): 81 return False 82 # Ignore value. 83 return default
84 85
86 -def TwistedWebErrorAsDict(self, request):
87 """Additional method for twisted.web.error.Error.""" 88 result = {} 89 result['http_error'] = self.status 90 result['response'] = self.response 91 return result
92 93
94 -def TwistedWebErrorPageAsDict(self, request):
95 """Additional method for twisted.web.error.Error.""" 96 result = {} 97 result['http_error'] = self.code 98 result['response'] = self.brief 99 result['detail'] = self.detail 100 return result
101 102 103 # Add .asDict() method to twisted.web.error.Error to simplify the code below. 104 error.Error.asDict = TwistedWebErrorAsDict 105 error.PageRedirect.asDict = TwistedWebErrorAsDict 106 error.ErrorPage.asDict = TwistedWebErrorPageAsDict 107 error.NoResource.asDict = TwistedWebErrorPageAsDict 108 error.ForbiddenResource.asDict = TwistedWebErrorPageAsDict 109 110
111 -def FilterOut(data):
112 """Returns a copy with None, False, "", [], () and {} removed. 113 Warning: converts tuple to list.""" 114 if isinstance(data, (list, tuple)): 115 # Recurse in every items and filter them out. 116 items = map(FilterOut, data) 117 if not filter(lambda x: not x in ('', False, None, [], {}, ()), items): 118 return None 119 return items 120 elif isinstance(data, dict): 121 return dict(filter(lambda x: not x[1] in ('', False, None, [], {}, ()), 122 [(k, FilterOut(v)) for (k, v) in data.iteritems()])) 123 else: 124 return data
125 126
127 -class JsonResource(resource.Resource):
128 """Base class for json data.""" 129 130 contentType = "application/json" 131 cache_seconds = 60 132 help = None 133 title = None 134 level = 0 135
136 - def __init__(self, status):
137 """Adds transparent lazy-child initialization.""" 138 resource.Resource.__init__(self) 139 # buildbot.status.builder.Status 140 self.status = status 141 if self.help: 142 title = '' 143 if self.title: 144 title = self.title + ' help' 145 self.putChild('help', 146 HelpResource(self.help, title=title, parent_node=self))
147
148 - def getChildWithDefault(self, path, request):
149 """Adds transparent support for url ending with /""" 150 if path == "" and len(request.postpath) == 0: 151 return self 152 # Equivalent to resource.Resource.getChildWithDefault() 153 if self.children.has_key(path): 154 return self.children[path] 155 return self.getChild(path, request)
156
157 - def putChild(self, name, res):
158 """Adds the resource's level for help links generation.""" 159 160 def RecurseFix(res, level): 161 res.level = level + 1 162 for c in res.children.itervalues(): 163 RecurseFix(c, res.level)
164 165 RecurseFix(res, self.level) 166 resource.Resource.putChild(self, name, res)
167
168 - def render_GET(self, request):
169 """Renders a HTTP GET at the http request level.""" 170 data = self.content(request) 171 if isinstance(data, unicode): 172 data = data.encode("utf-8") 173 if RequestArgToBool(request, 'as_text', False): 174 request.setHeader("content-type", 'text/plain') 175 else: 176 request.setHeader("content-type", self.contentType) 177 request.setHeader("content-disposition", 178 "attachment; filename=\"%s.json\"" % request.path) 179 # Make sure we get fresh pages. 180 if self.cache_seconds: 181 now = datetime.datetime.utcnow() 182 expires = now + datetime.timedelta(seconds=self.cache_seconds) 183 request.setHeader("Expires", 184 expires.strftime("%a, %d %b %Y %H:%M:%S GMT")) 185 request.setHeader("Pragma", "no-cache") 186 return data
187
188 - def content(self, request):
189 """Renders the json dictionaries.""" 190 # Implement filtering at global level and every child. 191 select = request.args.get('select') 192 if select is not None: 193 del request.args['select'] 194 # Do not render self.asDict()! 195 data = {} 196 # Remove superfluous / 197 select = [s.strip('/') for s in select] 198 select.sort(cmp=lambda x,y: cmp(x.count('/'), y.count('/')), 199 reverse=True) 200 for item in select: 201 # Start back at root. 202 node = data 203 # Implementation similar to twisted.web.resource.getChildForRequest 204 # but with a hacked up request. 205 child = self 206 prepath = request.prepath[:] 207 postpath = request.postpath[:] 208 request.postpath = filter(None, item.split('/')) 209 while request.postpath and not child.isLeaf: 210 pathElement = request.postpath.pop(0) 211 node[pathElement] = {} 212 node = node[pathElement] 213 request.prepath.append(pathElement) 214 child = child.getChildWithDefault(pathElement, request) 215 node.update(child.asDict(request)) 216 request.prepath = prepath 217 request.postpath = postpath 218 else: 219 data = self.asDict(request) 220 as_text = RequestArgToBool(request, 'as_text', False) 221 filter_out = RequestArgToBool(request, 'filter', as_text) 222 if filter_out: 223 data = FilterOut(data) 224 if RequestArgToBool(request, 'compact', not as_text): 225 return json.dumps(data, sort_keys=True, separators=(',',':')) 226 else: 227 return json.dumps(data, sort_keys=True, indent=2)
228
229 - def asDict(self, request):
230 """Generates the json dictionary. 231 232 By default, renders every childs.""" 233 if self.children: 234 data = {} 235 for name in self.children: 236 child = self.getChildWithDefault(name, request) 237 if isinstance(child, JsonResource): 238 data[name] = child.asDict(request) 239 # else silently pass over non-json resources. 240 return data 241 else: 242 raise NotImplementedError()
243 244
245 -def ToHtml(text):
246 """Convert a string in a wiki-style format into HTML.""" 247 indent = 0 248 in_item = False 249 output = [] 250 for line in text.splitlines(False): 251 match = re.match(r'^( +)\- (.*)$', line) 252 if match: 253 if indent < len(match.group(1)): 254 output.append('<ul>') 255 indent = len(match.group(1)) 256 elif indent > len(match.group(1)): 257 while indent > len(match.group(1)): 258 output.append('</ul>') 259 indent -= 2 260 if in_item: 261 # Close previous item 262 output.append('</li>') 263 output.append('<li>') 264 in_item = True 265 line = match.group(2) 266 elif indent: 267 if line.startswith((' ' * indent) + ' '): 268 # List continuation 269 line = line.strip() 270 else: 271 # List is done 272 if in_item: 273 output.append('</li>') 274 in_item = False 275 while indent > 0: 276 output.append('</ul>') 277 indent -= 2 278 279 if line.startswith('/'): 280 if not '?' in line: 281 line_full = line + '?as_text=1' 282 else: 283 line_full = line + '&as_text=1' 284 output.append('<a href="' + html.escape(line_full) + '">' + 285 html.escape(line) + '</a>') 286 else: 287 output.append(html.escape(line).replace(' ', '&nbsp;&nbsp;')) 288 if not in_item: 289 output.append('<br>') 290 291 if in_item: 292 output.append('</li>') 293 while indent > 0: 294 output.append('</ul>') 295 indent -= 2 296 return '\n'.join(output)
297 298
299 -class HelpResource(HtmlResource):
300 - def __init__(self, text, title, parent_node):
301 HtmlResource.__init__(self) 302 self.text = text 303 self.title = title 304 self.parent_node = parent_node
305
306 - def content(self, request, cxt):
307 cxt['level'] = self.parent_node.level 308 cxt['text'] = ToHtml(self.text) 309 cxt['children'] = [ n for n in self.parent_node.children.keys() if n != 'help' ] 310 cxt['flags'] = ToHtml(FLAGS) 311 cxt['examples'] = ToHtml(EXAMPLES).replace( 312 'href="/json', 313 'href="%sjson' % (self.level * '../')) 314 315 template = request.site.buildbot_service.templates.get_template("jsonhelp.html") 316 return template.render(**cxt)
317
318 -class BuilderJsonResource(JsonResource):
319 help = """Describe a single builder. 320 """ 321 title = 'Builder' 322
323 - def __init__(self, status, builder_status):
324 JsonResource.__init__(self, status) 325 self.builder_status = builder_status 326 self.putChild('builds', BuildsJsonResource(status, builder_status)) 327 self.putChild('slaves', BuilderSlavesJsonResources(status, 328 builder_status))
329
330 - def asDict(self, request):
331 # buildbot.status.builder.BuilderStatus 332 return self.builder_status.asDict()
333 334
335 -class BuildersJsonResource(JsonResource):
336 help = """List of all the builders defined on a master. 337 """ 338 title = 'Builders' 339
340 - def __init__(self, status):
341 JsonResource.__init__(self, status) 342 for builder_name in self.status.getBuilderNames(): 343 self.putChild(builder_name, 344 BuilderJsonResource(status, 345 status.getBuilder(builder_name)))
346 347
348 -class BuilderSlavesJsonResources(JsonResource):
349 help = """Describe the slaves attached to a single builder. 350 """ 351 title = 'BuilderSlaves' 352
353 - def __init__(self, status, builder_status):
354 JsonResource.__init__(self, status) 355 self.builder_status = builder_status 356 for slave_name in self.builder_status.slavenames: 357 self.putChild(slave_name, 358 SlaveJsonResource(status, 359 self.status.getSlave(slave_name)))
360 361
362 -class BuildJsonResource(JsonResource):
363 help = """Describe a single build. 364 """ 365 title = 'Build' 366
367 - def __init__(self, status, build_status):
374
375 - def asDict(self, request):
376 return self.build_status.asDict()
377 378
379 -class AllBuildsJsonResource(JsonResource):
380 help = """All the builds that were run on a builder. 381 """ 382 title = 'AllBuilds' 383
384 - def __init__(self, status, builder_status):
385 JsonResource.__init__(self, status) 386 self.builder_status = builder_status
387
388 - def getChild(self, path, request):
389 # Dynamic childs. 390 if isinstance(path, int) or _IS_INT.match(path): 391 build_status = self.builder_status.getBuild(int(path)) 392 if build_status: 393 build_status_number = str(build_status.getNumber()) 394 # Happens with negative numbers. 395 child = self.children.get(build_status_number) 396 if child: 397 return child 398 # Create it on-demand. 399 child = BuildJsonResource(self.status, build_status) 400 # Cache it. Never cache negative numbers. 401 # TODO(maruel): Cleanup the cache once it's too heavy! 402 self.putChild(build_status_number, child) 403 return child 404 return JsonResource.getChild(self, path, request)
405
406 - def asDict(self, request):
407 results = {} 408 # If max > buildCacheSize, it'll trash the cache... 409 max = int(RequestArg(request, 'max', 410 self.builder_status.buildCacheSize)) 411 for i in range(0, max): 412 child = self.getChildWithDefault(-i, request) 413 if not isinstance(child, BuildJsonResource): 414 continue 415 results[child.build_status.getNumber()] = child.asDict(request) 416 return results
417 418
419 -class BuildsJsonResource(AllBuildsJsonResource):
420 help = """Builds that were run on a builder. 421 """ 422 title = 'Builds' 423
424 - def __init__(self, status, builder_status):
425 AllBuildsJsonResource.__init__(self, status, builder_status) 426 self.putChild('_all', AllBuildsJsonResource(status, builder_status))
427
428 - def getChild(self, path, request):
429 # Transparently redirects to _all if path is not ''. 430 return self.children['_all'].getChildWithDefault(path, request)
431
432 - def asDict(self, request):
433 # This would load all the pickles and is way too heavy, especially that 434 # it would trash the cache: 435 # self.children['builds'].asDict(request) 436 # TODO(maruel) This list should also need to be cached but how? 437 builds = dict([ 438 (int(file), None) 439 for file in os.listdir(self.builder_status.basedir) 440 if _IS_INT.match(file) 441 ]) 442 return builds
443 444
445 -class BuildStepJsonResource(JsonResource):
446 help = """A single build step. 447 """ 448 title = 'BuildStep' 449
450 - def __init__(self, status, build_step_status):
451 # buildbot.status.builder.BuildStepStatus 452 JsonResource.__init__(self, status) 453 self.build_step_status = build_step_status
454 # TODO self.putChild('logs', LogsJsonResource()) 455
456 - def asDict(self, request):
457 return self.build_step_status.asDict()
458 459
460 -class BuildStepsJsonResource(JsonResource):
461 help = """A list of build steps that occurred during a build. 462 """ 463 title = 'BuildSteps' 464
465 - def __init__(self, status, build_status):
468 # The build steps are constantly changing until the build is done so 469 # keep a reference to build_status instead 470
471 - def getChild(self, path, request):
472 # Dynamic childs. 473 build_set_status = None 474 if isinstance(path, int) or _IS_INT.match(path): 475 build_set_status = self.build_status.getSteps[int(path)] 476 else: 477 steps_dict = dict([(step.getName(), step) 478 for step in self.build_status.getStep()]) 479 build_set_status = steps_dict.get(path) 480 if build_set_status: 481 # Create it on-demand. 482 child = BuildStepJsonResource(status, build_step_status) 483 # Cache it. 484 index = self.build_status.getSteps().index(build_step_status) 485 self.putChild(str(index), child) 486 self.putChild(build_set_status.getName(), child) 487 return child 488 return JsonResource.getChild(self, path, request)
489
490 - def asDict(self, request):
491 # Only use the number and not the names! 492 results = {} 493 index = 0 494 for step in self.build_status.getStep(): 495 results[index] = step 496 index += 1 497 return results
498 499
500 -class ChangeJsonResource(JsonResource):
501 help = """Describe a single change that originates from a change source. 502 """ 503 title = 'Change' 504
505 - def __init__(self, status, change):
506 # buildbot.changes.changes.Change 507 JsonResource.__init__(self, status) 508 self.change = change
509
510 - def asDict(self, request):
511 return self.change.asDict()
512 513
514 -class ChangesJsonResource(JsonResource):
515 help = """List of changes. 516 """ 517 title = 'Changes' 518
519 - def __init__(self, status, changes):
520 JsonResource.__init__(self, status) 521 for c in changes: 522 # TODO(maruel): Problem with multiple changes with the same number. 523 # Probably try server hack specific so we could fix it on this side 524 # instead. But there is still the problem with multiple pollers from 525 # different repo where the numbers could clash. 526 number = str(c.number) 527 while number in self.children: 528 # TODO(maruel): Do something better? 529 number = str(int(c.number)+1) 530 self.putChild(number, ChangeJsonResource(status, c))
531
532 - def asDict(self, request):
533 """Don't throw an exception when there is no child.""" 534 if not self.children: 535 return {} 536 return JsonResource.asDict(self, request)
537 538
539 -class ChangeSourcesJsonResource(JsonResource):
540 help = """Describe a change source. 541 """ 542 title = 'ChangeSources' 543
544 - def asDict(self, request):
545 result = {} 546 n = 0 547 for c in self.status.getChangeSources(): 548 # buildbot.changes.changes.ChangeMaster 549 change = {} 550 change['description'] = c.describe() 551 result[n] = change 552 n += 1 553 return result
554 555
556 -class ProjectJsonResource(JsonResource):
557 help = """Project-wide settings. 558 """ 559 title = 'Project' 560
561 - def asDict(self, request):
562 return self.status.asDict()
563 564
565 -class SlaveJsonResource(JsonResource):
566 help = """Describe a slave. 567 """ 568 title = 'Slave' 569
570 - def __init__(self, status, slave_status):
571 JsonResource.__init__(self, status) 572 self.slave_status = slave_status 573 self.name = self.slave_status.getName() 574 self.builders = None
575
576 - def getBuilders(self):
577 if self.builders is None: 578 # Figure out all the builders to which it's attached 579 self.builders = [] 580 for builderName in self.status.getBuilderNames(): 581 if self.name in self.status.getBuilder(builderName).slavenames: 582 self.builders.append(builderName) 583 return self.builders
584
585 - def asDict(self, request):
586 results = self.slave_status.asDict() 587 # Enhance it by adding more informations. 588 results['builders'] = {} 589 for builderName in self.getBuilders(): 590 builds = [] 591 builder_status = self.status.getBuilder(builderName) 592 for i in range(1, builder_status.buildCacheSize - 1): 593 build_status = builder_status.getBuild(-i) 594 if not build_status or not build_status.isFinished(): 595 # If not finished, it will appear in runningBuilds. 596 break 597 if build_status.getSlavename() == self.name: 598 builds.append(build_status.getNumber()) 599 results['builders'][builderName] = builds 600 return results
601 602
603 -class SlavesJsonResource(JsonResource):
604 help = """List the registered slaves. 605 """ 606 title = 'Slaves' 607
608 - def __init__(self, status):
609 JsonResource.__init__(self, status) 610 for slave_name in status.getSlaveNames(): 611 self.putChild(slave_name, 612 SlaveJsonResource(status, 613 status.getSlave(slave_name)))
614 615
616 -class SourceStampJsonResource(JsonResource):
617 help = """Describe the sources for a BuildRequest. 618 """ 619 title = 'SourceStamp' 620
621 - def __init__(self, status, source_stamp):
622 # buildbot.sourcestamp.SourceStamp 623 JsonResource.__init__(self, status) 624 self.source_stamp = source_stamp 625 self.putChild('changes', 626 ChangesJsonResource(status, source_stamp.changes))
627 # TODO(maruel): Should redirect to the patch's url instead. 628 #if source_stamp.patch: 629 # self.putChild('patch', StaticHTML(source_stamp.path)) 630
631 - def asDict(self, request):
632 return self.source_stamp.asDict()
633 634
635 -class JsonStatusResource(JsonResource):
636 """Retrieves all json data.""" 637 help = """JSON status 638 639 Root page to give a fair amount of information in the current buildbot master 640 status. You may want to use a child instead to reduce the load on the server. 641 642 For help on any sub directory, use url /child/help 643 """ 644 title = 'Buildbot JSON' 645
646 - def __init__(self, status):
647 JsonResource.__init__(self, status) 648 self.level = 1 649 self.putChild('builders', BuildersJsonResource(status)) 650 self.putChild('change_sources', ChangeSourcesJsonResource(status)) 651 self.putChild('project', ProjectJsonResource(status)) 652 self.putChild('slaves', SlavesJsonResource(status)) 653 # This needs to be called before the first HelpResource().body call. 654 self.hackExamples()
655
656 - def content(self, request):
657 result = JsonResource.content(self, request) 658 # This is done to hook the downloaded filename. 659 request.path = 'buildbot' 660 return result
661
662 - def hackExamples(self):
663 global EXAMPLES 664 # Find the first builder with a previous build or select the last one. 665 builder = None 666 for b in self.status.getBuilderNames(): 667 builder = self.status.getBuilder(b) 668 if builder.getBuild(-1): 669 break 670 if not builder: 671 return 672 EXAMPLES = EXAMPLES.replace('<A_BUILDER>', builder.getName()) 673 build = builder.getBuild(-1) 674 if build: 675 EXAMPLES = EXAMPLES.replace('<A_BUILD>', str(build.getNumber())) 676 if builder.slavenames: 677 EXAMPLES = EXAMPLES.replace('<A_SLAVE>', builder.slavenames[0])
678 679 # vim: set ts=4 sts=4 sw=4 et: 680