buildbot.status.web.console

92 """Main console class. It displays a user-oriented status page. 93 Every change is a line in the page, and it shows the result of the first 94 build with this change for each slave.""" 95

96 - def __init__(self, orderByTime=False):

97 HtmlResource.__init__(self) 98 99 self.status = None 100 101 if orderByTime: 102 self.comparator = TimeRevisionComparator() 103 else: 104 self.comparator = IntegerRevisionComparator()

105

106 - def getPageTitle(self, request):

107 status = self.getStatus(request) 108 title = status.getTitle() 109 if title: 110 return "BuildBot: %s" % title 111 else: 112 return "BuildBot"

113

114 - def getChangeManager(self, request):

115 return request.site.buildbot_service.parent.change_svc

116 117 ## 118 ## Data gathering functions 119 ## 120

121 - def getHeadBuild(self, builder):

122 """Get the most recent build for the given builder. 123 """ 124 build = builder.getBuild(-1) 125 126 # HACK: Work around #601, the head build may be None if it is 127 # locked. 128 if build is None: 129 build = builder.getBuild(-2) 130 131 return build

132

133 - def fetchChangesFromHistory(self, status, max_depth, max_builds, debugInfo):

134 """Look at the history of the builders and try to fetch as many changes 135 as possible. We need this when the main source does not contain enough 136 sourcestamps. 137 138 max_depth defines how many builds we will parse for a given builder. 139 max_builds defines how many builds total we want to parse. This is to 140 limit the amount of time we spend in this function. 141 142 This function is sub-optimal, but the information returned by this 143 function is cached, so this function won't be called more than once. 144 """ 145 146 allChanges = list() 147 build_count = 0 148 for builderName in status.getBuilderNames()[:]: 149 if build_count > max_builds: 150 break 151 152 builder = status.getBuilder(builderName) 153 build = self.getHeadBuild(builder) 154 depth = 0 155 while build and depth < max_depth and build_count < max_builds: 156 depth += 1 157 build_count += 1 158 sourcestamp = build.getSourceStamp() 159 allChanges.extend(sourcestamp.changes[:]) 160 build = build.getPreviousBuild() 161 162 debugInfo["source_fetch_len"] = len(allChanges) 163 return allChanges

164 165 @defer.deferredGenerator

166 - def getAllChanges(self, request, status, debugInfo):

167 master = request.site.buildbot_service.master 168 169 wfd = defer.waitForDeferred( 170 master.db.changes.getRecentChanges(25)) 171 yield wfd 172 chdicts = wfd.getResult() 173 174 # convert those to Change instances 175 wfd = defer.waitForDeferred( 176 defer.gatherResults([ 177 changes.Change.fromChdict(master, chdict) 178 for chdict in chdicts ])) 179 yield wfd 180 allChanges = wfd.getResult() 181 182 allChanges.sort(key=self.comparator.getSortingKey()) 183 184 # Remove the dups 185 prevChange = None 186 newChanges = [] 187 for change in allChanges: 188 rev = change.revision 189 if not prevChange or rev != prevChange.revision: 190 newChanges.append(change) 191 prevChange = change 192 allChanges = newChanges 193 194 yield allChanges

195

196 - def getBuildDetails(self, request, builderName, build):

197 """Returns an HTML list of failures for a given build.""" 198 details = {} 199 if not build.getLogs(): 200 return details 201 202 for step in build.getSteps(): 203 (result, reason) = step.getResults() 204 if result == builder.FAILURE: 205 name = step.getName() 206 207 # Remove html tags from the error text. 208 stripHtml = re.compile(r'<.*?>') 209 strippedDetails = stripHtml.sub('', ' '.join(step.getText())) 210 211 details['buildername'] = builderName 212 details['status'] = strippedDetails 213 details['reason'] = reason 214 logs = details['logs'] = [] 215 216 if step.getLogs(): 217 for log in step.getLogs(): 218 logname = log.getName() 219 logurl = request.childLink( 220 "../builders/%s/builds/%s/steps/%s/logs/%s" % 221 (urllib.quote(builderName), 222 build.getNumber(), 223 urllib.quote(name), 224 urllib.quote(logname))) 225 logs.append(dict(url=logurl, name=logname)) 226 return details

227

228 - def getBuildsForRevision(self, request, builder, builderName, lastRevision, 229 numBuilds, debugInfo):

230 """Return the list of all the builds for a given builder that we will 231 need to be able to display the console page. We start by the most recent 232 build, and we go down until we find a build that was built prior to the 233 last change we are interested in.""" 234 235 revision = lastRevision 236 237 builds = [] 238 build = self.getHeadBuild(builder) 239 number = 0 240 while build and number < numBuilds: 241 debugInfo["builds_scanned"] += 1 242 number += 1 243 244 # Get the last revision in this build. 245 # We first try "got_revision", but if it does not work, then 246 # we try "revision". 247 got_rev = -1 248 try: 249 got_rev = build.getProperty("got_revision") 250 if not self.comparator.isValidRevision(got_rev): 251 got_rev = -1 252 except KeyError: 253 pass 254 255 try: 256 if got_rev == -1: 257 got_rev = build.getProperty("revision") 258 if not self.comparator.isValidRevision(got_rev): 259 got_rev = -1 260 except: 261 pass 262 263 # We ignore all builds that don't have last revisions. 264 # TODO(nsylvain): If the build is over, maybe it was a problem 265 # with the update source step. We need to find a way to tell the 266 # user that his change might have broken the source update. 267 if got_rev and got_rev != -1: 268 details = self.getBuildDetails(request, builderName, build) 269 devBuild = DevBuild(got_rev, build, details) 270 builds.append(devBuild) 271 272 # Now break if we have enough builds. 273 current_revision = self.getChangeForBuild( 274 build, revision) 275 if self.comparator.isRevisionEarlier( 276 devBuild, current_revision): 277 break 278 279 build = build.getPreviousBuild() 280 281 return builds

282

283 - def getChangeForBuild(self, build, revision):

284 if not build or not build.getChanges(): # Forced build 285 return DevBuild(revision, build, None) 286 287 for change in build.getChanges(): 288 if change.revision == revision: 289 return change 290 291 # No matching change, return the last change in build. 292 changes = list(build.getChanges()) 293 changes.sort(key=self.comparator.getSortingKey()) 294 return changes[-1]

295

296 - def getAllBuildsForRevision(self, status, request, lastRevision, numBuilds, 297 categories, builders, debugInfo):

298 """Returns a dictionary of builds we need to inspect to be able to 299 display the console page. The key is the builder name, and the value is 300 an array of build we care about. We also returns a dictionary of 301 builders we care about. The key is it's category. 302 303 lastRevision is the last revision we want to display in the page. 304 categories is a list of categories to display. It is coming from the 305 HTTP GET parameters. 306 builders is a list of builders to display. It is coming from the HTTP 307 GET parameters. 308 """ 309 310 allBuilds = dict() 311 312 # List of all builders in the dictionary. 313 builderList = dict() 314 315 debugInfo["builds_scanned"] = 0 316 # Get all the builders. 317 builderNames = status.getBuilderNames()[:] 318 for builderName in builderNames: 319 builder = status.getBuilder(builderName) 320 321 # Make sure we are interested in this builder. 322 if categories and builder.category not in categories: 323 continue 324 if builders and builderName not in builders: 325 continue 326 327 # We want to display this builder. 328 category = builder.category or "default" 329 # Strip the category to keep only the text before the first |. 330 # This is a hack to support the chromium usecase where they have 331 # multiple categories for each slave. We use only the first one. 332 # TODO(nsylvain): Create another way to specify "display category" 333 # in master.cfg. 334 category = category.split('|')[0] 335 if not builderList.get(category): 336 builderList[category] = [] 337 338 # Append this builder to the dictionary of builders. 339 builderList[category].append(builderName) 340 # Set the list of builds for this builder. 341 allBuilds[builderName] = self.getBuildsForRevision(request, 342 builder, 343 builderName, 344 lastRevision, 345 numBuilds, 346 debugInfo) 347 348 return (builderList, allBuilds)

349 350 351 ## 352 ## Display functions 353 ## 354

355 - def displayCategories(self, builderList, debugInfo):

356 """Display the top category line.""" 357 358 count = 0 359 for category in builderList: 360 count += len(builderList[category]) 361 362 categories = builderList.keys() 363 categories.sort() 364 365 cs = [] 366 367 for category in categories: 368 c = {} 369 370 c["name"] = category 371 372 # To be able to align the table correctly, we need to know 373 # what percentage of space this category will be taking. This is 374 # (#Builders in Category) / (#Builders Total) * 100. 375 c["size"] = (len(builderList[category]) * 100) / count 376 cs.append(c) 377 378 return cs

379

380 - def displaySlaveLine(self, status, builderList, debugInfo):

381 """Display a line the shows the current status for all the builders we 382 care about.""" 383 384 nbSlaves = 0 385 386 # Get the number of builders. 387 for category in builderList: 388 nbSlaves += len(builderList[category]) 389 390 # Get the categories, and order them alphabetically. 391 categories = builderList.keys() 392 categories.sort() 393 394 slaves = {} 395 396 # For each category, we display each builder. 397 for category in categories: 398 slaves[category] = [] 399 # For each builder in this category, we set the build info and we 400 # display the box. 401 for builder in builderList[category]: 402 s = {} 403 s["color"] = "notstarted" 404 s["pageTitle"] = builder 405 s["url"] = "./builders/%s" % urllib.quote(builder) 406 state, builds = status.getBuilder(builder).getState() 407 # Check if it's offline, if so, the box is purple. 408 if state == "offline": 409 s["color"] = "offline" 410 else: 411 # If not offline, then display the result of the last 412 # finished build. 413 build = self.getHeadBuild(status.getBuilder(builder)) 414 while build and not build.isFinished(): 415 build = build.getPreviousBuild() 416 417 if build: 418 s["color"] = getResultsClass(build.getResults(), None, 419 False) 420 421 slaves[category].append(s) 422 423 return slaves

424

425 - def displayStatusLine(self, builderList, allBuilds, revision, debugInfo):

426 """Display the boxes that represent the status of each builder in the 427 first build "revision" was in. Returns an HTML list of errors that 428 happened during these builds.""" 429 430 details = [] 431 nbSlaves = 0 432 for category in builderList: 433 nbSlaves += len(builderList[category]) 434 435 # Sort the categories. 436 categories = builderList.keys() 437 categories.sort() 438 439 builds = {} 440 441 # Display the boxes by category group. 442 for category in categories: 443 444 builds[category] = [] 445 446 # Display the boxes for each builder in this category. 447 for builder in builderList[category]: 448 introducedIn = None 449 firstNotIn = None 450 451 # Find the first build that does not include the revision. 452 for build in allBuilds[builder]: 453 if self.comparator.isRevisionEarlier(build, revision): 454 firstNotIn = build 455 break 456 else: 457 introducedIn = build 458 459 # Get the results of the first build with the revision, and the 460 # first build that does not include the revision. 461 results = None 462 previousResults = None 463 if introducedIn: 464 results = introducedIn.results 465 if firstNotIn: 466 previousResults = firstNotIn.results 467 468 isRunning = False 469 if introducedIn and not introducedIn.isFinished: 470 isRunning = True 471 472 url = "./waterfall" 473 pageTitle = builder 474 tag = "" 475 current_details = {} 476 if introducedIn: 477 current_details = introducedIn.details or "" 478 url = "./buildstatus?builder=%s&number=%s" % (urllib.quote(builder), 479 introducedIn.number) 480 pageTitle += " " 481 pageTitle += urllib.quote(' '.join(introducedIn.text), ' \n\\/:') 482 483 builderStrip = builder.replace(' ', '') 484 builderStrip = builderStrip.replace('(', '') 485 builderStrip = builderStrip.replace(')', '') 486 builderStrip = builderStrip.replace('.', '') 487 tag = "Tag%s%s" % (builderStrip, introducedIn.number) 488 489 if isRunning: 490 pageTitle += ' ETA: %ds' % (introducedIn.eta or 0) 491 492 resultsClass = getResultsClass(results, previousResults, isRunning) 493 494 b = {} 495 b["url"] = url 496 b["pageTitle"] = pageTitle 497 b["color"] = resultsClass 498 b["tag"] = tag 499 500 builds[category].append(b) 501 502 # If the box is red, we add the explaination in the details 503 # section. 504 if current_details and resultsClass == "failure": 505 details.append(current_details) 506 507 return (builds, details)

508

509 - def filterRevisions(self, revisions, filter=None, max_revs=None):

510 """Filter a set of revisions based on any number of filter criteria. 511 If specified, filter should be a dict with keys corresponding to 512 revision attributes, and values of 1+ strings""" 513 if not filter: 514 if max_revs is None: 515 for rev in reversed(revisions): 516 yield DevRevision(rev) 517 else: 518 for index,rev in enumerate(reversed(revisions)): 519 if index >= max_revs: 520 break 521 yield DevRevision(rev) 522 else: 523 for index, rev in enumerate(reversed(revisions)): 524 if max_revs and index >= max_revs: 525 break 526 try: 527 for field,acceptable in filter.iteritems(): 528 if not hasattr(rev, field): 529 raise DoesNotPassFilter 530 if type(acceptable) in (str, unicode): 531 if getattr(rev, field) != acceptable: 532 raise DoesNotPassFilter 533 elif type(acceptable) in (list, tuple, set): 534 if getattr(rev, field) not in acceptable: 535 raise DoesNotPassFilter 536 yield DevRevision(rev) 537 except DoesNotPassFilter: 538 pass

539

540 - def displayPage(self, request, status, builderList, allBuilds, revisions, 541 categories, repository, branch, debugInfo):

542 """Display the console page.""" 543 # Build the main template directory with all the informations we have. 544 subs = dict() 545 subs["branch"] = branch or 'trunk' 546 subs["repository"] = repository 547 if categories: 548 subs["categories"] = ' '.join(categories) 549 subs["time"] = time.strftime("%a %d %b %Y %H:%M:%S", 550 time.localtime(util.now())) 551 subs["debugInfo"] = debugInfo 552 subs["ANYBRANCH"] = ANYBRANCH 553 554 if builderList: 555 subs["categories"] = self.displayCategories(builderList, debugInfo) 556 subs['slaves'] = self.displaySlaveLine(status, builderList, debugInfo) 557 else: 558 subs["categories"] = [] 559 560 subs['revisions'] = [] 561 562 # For each revision we show one line 563 for revision in revisions: 564 r = {} 565 566 # Fill the dictionary with this new information 567 r['id'] = revision.revision 568 r['link'] = revision.revlink 569 r['who'] = revision.who 570 r['date'] = revision.date 571 r['comments'] = revision.comments 572 r['repository'] = revision.repository 573 r['project'] = revision.project 574 575 # Display the status for all builders. 576 (builds, details) = self.displayStatusLine(builderList, 577 allBuilds, 578 revision, 579 debugInfo) 580 r['builds'] = builds 581 r['details'] = details 582 583 # Calculate the td span for the comment and the details. 584 r["span"] = len(builderList) + 2 585 586 subs['revisions'].append(r) 587 588 # 589 # Display the footer of the page. 590 # 591 debugInfo["load_time"] = time.time() - debugInfo["load_time"] 592 return subs

593 594

595 - def content(self, request, cxt):

596 "This method builds the main console view display." 597 598 reload_time = None 599 # Check if there was an arg. Don't let people reload faster than 600 # every 15 seconds. 0 means no reload. 601 if "reload" in request.args: 602 try: 603 reload_time = int(request.args["reload"][0]) 604 if reload_time != 0: 605 reload_time = max(reload_time, 15) 606 except ValueError: 607 pass 608 609 request.setHeader('Cache-Control', 'no-cache') 610 611 # Sets the default reload time to 60 seconds. 612 if not reload_time: 613 reload_time = 60 614 615 # Append the tag to refresh the page. 616 if reload_time is not None and reload_time != 0: 617 cxt['refresh'] = reload_time 618 619 # Debug information to display at the end of the page. 620 debugInfo = cxt['debuginfo'] = dict() 621 debugInfo["load_time"] = time.time() 622 623 # get url parameters 624 # Categories to show information for. 625 categories = request.args.get("category", []) 626 # List of all builders to show on the page. 627 builders = request.args.get("builder", []) 628 # Repo used to filter the changes shown. 629 repository = request.args.get("repository", [None])[0] 630 # Branch used to filter the changes shown. 631 branch = request.args.get("branch", [ANYBRANCH])[0] 632 # List of all the committers name to display on the page. 633 devName = request.args.get("name", []) 634 635 # and the data we want to render 636 status = self.getStatus(request) 637 638 # Keep only the revisions we care about. 639 # By default we process the last 40 revisions. 640 # If a dev name is passed, we look for the changes by this person in the 641 # last 80 revisions. 642 numRevs = int(request.args.get("revs", [40])[0]) 643 if devName: 644 numRevs *= 2 645 numBuilds = numRevs 646 647 # Get all changes we can find. This is a DB operation, so it must use 648 # a deferred. 649 d = self.getAllChanges(request, status, debugInfo) 650 def got_changes(allChanges): 651 debugInfo["source_all"] = len(allChanges) 652 653 revFilter = {} 654 if branch != ANYBRANCH: 655 revFilter['branch'] = branch 656 if devName: 657 revFilter['who'] = devName 658 if repository: 659 revFilter['repository'] = repository 660 revisions = list(self.filterRevisions(allChanges, max_revs=numRevs, 661 filter=revFilter)) 662 debugInfo["revision_final"] = len(revisions) 663 664 # Fetch all the builds for all builders until we get the next build 665 # after lastRevision. 666 builderList = None 667 allBuilds = None 668 if revisions: 669 lastRevision = revisions[len(revisions) - 1].revision 670 debugInfo["last_revision"] = lastRevision 671 672 (builderList, allBuilds) = self.getAllBuildsForRevision(status, 673 request, 674 lastRevision, 675 numBuilds, 676 categories, 677 builders, 678 debugInfo) 679 680 debugInfo["added_blocks"] = 0 681 682 cxt.update(self.displayPage(request, status, builderList, 683 allBuilds, revisions, categories, 684 repository, branch, debugInfo)) 685 686 templates = request.site.buildbot_service.templates 687 template = templates.get_template("console.html") 688 data = template.render(cxt) 689 return data

690 d.addCallback(got_changes) 691 return d

Source Code for Module buildbot.status.web.console