buildbot.status.web.console

92 """Main console class. It displays a user-oriented status page. 93 Every change is a line in the page, and it shows the result of the first 94 build with this change for each slave.""" 95

96 - def __init__(self, orderByTime=False):

97 HtmlResource.__init__(self) 98 99 self.status = None 100 101 if orderByTime: 102 self.comparator = TimeRevisionComparator() 103 else: 104 self.comparator = IntegerRevisionComparator()

105

106 - def getPageTitle(self, request):

107 status = self.getStatus(request) 108 title = status.getTitle() 109 if title: 110 return "BuildBot: %s" % title 111 else: 112 return "BuildBot"

113

114 - def getChangeManager(self, request):

115 return request.site.buildbot_service.parent.change_svc

116 117 ## 118 ## Data gathering functions 119 ## 120

121 - def getHeadBuild(self, builder):

122 """Get the most recent build for the given builder. 123 """ 124 build = builder.getBuild(-1) 125 126 # HACK: Work around #601, the head build may be None if it is 127 # locked. 128 if build is None: 129 build = builder.getBuild(-2) 130 131 return build

132

133 - def fetchChangesFromHistory(self, status, max_depth, max_builds, debugInfo):

134 """Look at the history of the builders and try to fetch as many changes 135 as possible. We need this when the main source does not contain enough 136 sourcestamps. 137 138 max_depth defines how many builds we will parse for a given builder. 139 max_builds defines how many builds total we want to parse. This is to 140 limit the amount of time we spend in this function. 141 142 This function is sub-optimal, but the information returned by this 143 function is cached, so this function won't be called more than once. 144 """ 145 146 allChanges = list() 147 build_count = 0 148 for builderName in status.getBuilderNames()[:]: 149 if build_count > max_builds: 150 break 151 152 builder = status.getBuilder(builderName) 153 build = self.getHeadBuild(builder) 154 depth = 0 155 while build and depth < max_depth and build_count < max_builds: 156 depth += 1 157 build_count += 1 158 sourcestamp = build.getSourceStamp() 159 allChanges.extend(sourcestamp.changes[:]) 160 build = build.getPreviousBuild() 161 162 debugInfo["source_fetch_len"] = len(allChanges) 163 return allChanges

164 165 @defer.deferredGenerator

166 - def getAllChanges(self, request, status, debugInfo):

167 master = request.site.buildbot_service.master 168 169 wfd = defer.waitForDeferred( 170 master.db.changes.getRecentChanges(25)) 171 yield wfd 172 chdicts = wfd.getResult() 173 174 # convert those to Change instances 175 wfd = defer.waitForDeferred( 176 defer.gatherResults([ 177 changes.Change.fromChdict(master, chdict) 178 for chdict in chdicts ])) 179 yield wfd 180 allChanges = wfd.getResult() 181 182 allChanges.sort(key=self.comparator.getSortingKey()) 183 184 # Remove the dups 185 prevChange = None 186 newChanges = [] 187 for change in allChanges: 188 rev = change.revision 189 if not prevChange or rev != prevChange.revision: 190 newChanges.append(change) 191 prevChange = change 192 allChanges = newChanges 193 194 yield allChanges

195

196 - def getBuildDetails(self, request, builderName, build):

197 """Returns an HTML list of failures for a given build.""" 198 details = {} 199 if not build.getLogs(): 200 return details 201 202 for step in build.getSteps(): 203 (result, reason) = step.getResults() 204 if result == builder.FAILURE: 205 name = step.getName() 206 207 # Remove html tags from the error text. 208 stripHtml = re.compile(r'<.*?>') 209 strippedDetails = stripHtml.sub('', ' '.join(step.getText())) 210 211 details['buildername'] = builderName 212 details['status'] = strippedDetails 213 details['reason'] = reason 214 logs = details['logs'] = [] 215 216 if step.getLogs(): 217 for log in step.getLogs(): 218 logname = log.getName() 219 logurl = request.childLink( 220 "../builders/%s/builds/%s/steps/%s/logs/%s" % 221 (urllib.quote(builderName), 222 build.getNumber(), 223 urllib.quote(name), 224 urllib.quote(logname))) 225 logs.append(dict(url=logurl, name=logname)) 226 return details

227

228 - def getBuildsForRevision(self, request, builder, builderName, lastRevision, 229 numBuilds, debugInfo):

230 """Return the list of all the builds for a given builder that we will 231 need to be able to display the console page. We start by the most recent 232 build, and we go down until we find a build that was built prior to the 233 last change we are interested in.""" 234 235 revision = lastRevision 236 237 builds = [] 238 build = self.getHeadBuild(builder) 239 number = 0 240 while build and number < numBuilds: 241 debugInfo["builds_scanned"] += 1 242 number += 1 243 244 # Get the last revision in this build. 245 # We first try "got_revision", but if it does not work, then 246 # we try "revision". 247 got_rev = build.getProperty("got_revision", build.getProperty("revision", -1)) 248 if got_rev != -1 and not self.comparator.isValidRevision(got_rev): 249 got_rev = -1 250 251 # We ignore all builds that don't have last revisions. 252 # TODO(nsylvain): If the build is over, maybe it was a problem 253 # with the update source step. We need to find a way to tell the 254 # user that his change might have broken the source update. 255 if got_rev != -1: 256 details = self.getBuildDetails(request, builderName, build) 257 devBuild = DevBuild(got_rev, build, details) 258 builds.append(devBuild) 259 260 # Now break if we have enough builds. 261 current_revision = self.getChangeForBuild( 262 build, revision) 263 if self.comparator.isRevisionEarlier( 264 devBuild, current_revision): 265 break 266 267 build = build.getPreviousBuild() 268 269 return builds

270

271 - def getChangeForBuild(self, build, revision):

272 if not build or not build.getChanges(): # Forced build 273 return DevBuild(revision, build, None) 274 275 for change in build.getChanges(): 276 if change.revision == revision: 277 return change 278 279 # No matching change, return the last change in build. 280 changes = list(build.getChanges()) 281 changes.sort(key=self.comparator.getSortingKey()) 282 return changes[-1]

283

284 - def getAllBuildsForRevision(self, status, request, lastRevision, numBuilds, 285 categories, builders, debugInfo):

286 """Returns a dictionary of builds we need to inspect to be able to 287 display the console page. The key is the builder name, and the value is 288 an array of build we care about. We also returns a dictionary of 289 builders we care about. The key is it's category. 290 291 lastRevision is the last revision we want to display in the page. 292 categories is a list of categories to display. It is coming from the 293 HTTP GET parameters. 294 builders is a list of builders to display. It is coming from the HTTP 295 GET parameters. 296 """ 297 298 allBuilds = dict() 299 300 # List of all builders in the dictionary. 301 builderList = dict() 302 303 debugInfo["builds_scanned"] = 0 304 # Get all the builders. 305 builderNames = status.getBuilderNames()[:] 306 for builderName in builderNames: 307 builder = status.getBuilder(builderName) 308 309 # Make sure we are interested in this builder. 310 if categories and builder.category not in categories: 311 continue 312 if builders and builderName not in builders: 313 continue 314 315 # We want to display this builder. 316 category = builder.category or "default" 317 # Strip the category to keep only the text before the first |. 318 # This is a hack to support the chromium usecase where they have 319 # multiple categories for each slave. We use only the first one. 320 # TODO(nsylvain): Create another way to specify "display category" 321 # in master.cfg. 322 category = category.split('|')[0] 323 if not builderList.get(category): 324 builderList[category] = [] 325 326 # Append this builder to the dictionary of builders. 327 builderList[category].append(builderName) 328 # Set the list of builds for this builder. 329 allBuilds[builderName] = self.getBuildsForRevision(request, 330 builder, 331 builderName, 332 lastRevision, 333 numBuilds, 334 debugInfo) 335 336 return (builderList, allBuilds)

337 338 339 ## 340 ## Display functions 341 ## 342

343 - def displayCategories(self, builderList, debugInfo):

344 """Display the top category line.""" 345 346 count = 0 347 for category in builderList: 348 count += len(builderList[category]) 349 350 categories = builderList.keys() 351 categories.sort() 352 353 cs = [] 354 355 for category in categories: 356 c = {} 357 358 c["name"] = category 359 360 # To be able to align the table correctly, we need to know 361 # what percentage of space this category will be taking. This is 362 # (#Builders in Category) / (#Builders Total) * 100. 363 c["size"] = (len(builderList[category]) * 100) / count 364 cs.append(c) 365 366 return cs

367

368 - def displaySlaveLine(self, status, builderList, debugInfo):

369 """Display a line the shows the current status for all the builders we 370 care about.""" 371 372 nbSlaves = 0 373 374 # Get the number of builders. 375 for category in builderList: 376 nbSlaves += len(builderList[category]) 377 378 # Get the categories, and order them alphabetically. 379 categories = builderList.keys() 380 categories.sort() 381 382 slaves = {} 383 384 # For each category, we display each builder. 385 for category in categories: 386 slaves[category] = [] 387 # For each builder in this category, we set the build info and we 388 # display the box. 389 for builder in builderList[category]: 390 s = {} 391 s["color"] = "notstarted" 392 s["pageTitle"] = builder 393 s["url"] = "./builders/%s" % urllib.quote(builder) 394 state, builds = status.getBuilder(builder).getState() 395 # Check if it's offline, if so, the box is purple. 396 if state == "offline": 397 s["color"] = "offline" 398 else: 399 # If not offline, then display the result of the last 400 # finished build. 401 build = self.getHeadBuild(status.getBuilder(builder)) 402 while build and not build.isFinished(): 403 build = build.getPreviousBuild() 404 405 if build: 406 s["color"] = getResultsClass(build.getResults(), None, 407 False) 408 409 slaves[category].append(s) 410 411 return slaves

412

413 - def displayStatusLine(self, builderList, allBuilds, revision, debugInfo):

414 """Display the boxes that represent the status of each builder in the 415 first build "revision" was in. Returns an HTML list of errors that 416 happened during these builds.""" 417 418 details = [] 419 nbSlaves = 0 420 for category in builderList: 421 nbSlaves += len(builderList[category]) 422 423 # Sort the categories. 424 categories = builderList.keys() 425 categories.sort() 426 427 builds = {} 428 429 # Display the boxes by category group. 430 for category in categories: 431 432 builds[category] = [] 433 434 # Display the boxes for each builder in this category. 435 for builder in builderList[category]: 436 introducedIn = None 437 firstNotIn = None 438 439 # Find the first build that does not include the revision. 440 for build in allBuilds[builder]: 441 if self.comparator.isRevisionEarlier(build, revision): 442 firstNotIn = build 443 break 444 else: 445 introducedIn = build 446 447 # Get the results of the first build with the revision, and the 448 # first build that does not include the revision. 449 results = None 450 previousResults = None 451 if introducedIn: 452 results = introducedIn.results 453 if firstNotIn: 454 previousResults = firstNotIn.results 455 456 isRunning = False 457 if introducedIn and not introducedIn.isFinished: 458 isRunning = True 459 460 url = "./waterfall" 461 pageTitle = builder 462 tag = "" 463 current_details = {} 464 if introducedIn: 465 current_details = introducedIn.details or "" 466 url = "./buildstatus?builder=%s&number=%s" % (urllib.quote(builder), 467 introducedIn.number) 468 pageTitle += " " 469 pageTitle += urllib.quote(' '.join(introducedIn.text), ' \n\\/:') 470 471 builderStrip = builder.replace(' ', '') 472 builderStrip = builderStrip.replace('(', '') 473 builderStrip = builderStrip.replace(')', '') 474 builderStrip = builderStrip.replace('.', '') 475 tag = "Tag%s%s" % (builderStrip, introducedIn.number) 476 477 if isRunning: 478 pageTitle += ' ETA: %ds' % (introducedIn.eta or 0) 479 480 resultsClass = getResultsClass(results, previousResults, isRunning) 481 482 b = {} 483 b["url"] = url 484 b["pageTitle"] = pageTitle 485 b["color"] = resultsClass 486 b["tag"] = tag 487 488 builds[category].append(b) 489 490 # If the box is red, we add the explaination in the details 491 # section. 492 if current_details and resultsClass == "failure": 493 details.append(current_details) 494 495 return (builds, details)

496

497 - def filterRevisions(self, revisions, filter=None, max_revs=None):

498 """Filter a set of revisions based on any number of filter criteria. 499 If specified, filter should be a dict with keys corresponding to 500 revision attributes, and values of 1+ strings""" 501 if not filter: 502 if max_revs is None: 503 for rev in reversed(revisions): 504 yield DevRevision(rev) 505 else: 506 for index,rev in enumerate(reversed(revisions)): 507 if index >= max_revs: 508 break 509 yield DevRevision(rev) 510 else: 511 for index, rev in enumerate(reversed(revisions)): 512 if max_revs and index >= max_revs: 513 break 514 try: 515 for field,acceptable in filter.iteritems(): 516 if not hasattr(rev, field): 517 raise DoesNotPassFilter 518 if type(acceptable) in (str, unicode): 519 if getattr(rev, field) != acceptable: 520 raise DoesNotPassFilter 521 elif type(acceptable) in (list, tuple, set): 522 if getattr(rev, field) not in acceptable: 523 raise DoesNotPassFilter 524 yield DevRevision(rev) 525 except DoesNotPassFilter: 526 pass

527

528 - def displayPage(self, request, status, builderList, allBuilds, revisions, 529 categories, repository, project, branch, debugInfo):

530 """Display the console page.""" 531 # Build the main template directory with all the informations we have. 532 subs = dict() 533 subs["branch"] = branch or 'trunk' 534 subs["repository"] = repository 535 subs["project"] = project 536 if categories: 537 subs["categories"] = ' '.join(categories) 538 subs["time"] = time.strftime("%a %d %b %Y %H:%M:%S", 539 time.localtime(util.now())) 540 subs["debugInfo"] = debugInfo 541 subs["ANYBRANCH"] = ANYBRANCH 542 543 if builderList: 544 subs["categories"] = self.displayCategories(builderList, debugInfo) 545 subs['slaves'] = self.displaySlaveLine(status, builderList, debugInfo) 546 else: 547 subs["categories"] = [] 548 549 subs['revisions'] = [] 550 551 # For each revision we show one line 552 for revision in revisions: 553 r = {} 554 555 # Fill the dictionary with this new information 556 r['id'] = revision.revision 557 r['link'] = revision.revlink 558 r['who'] = revision.who 559 r['date'] = revision.date 560 r['comments'] = revision.comments 561 r['repository'] = revision.repository 562 r['project'] = revision.project 563 564 # Display the status for all builders. 565 (builds, details) = self.displayStatusLine(builderList, 566 allBuilds, 567 revision, 568 debugInfo) 569 r['builds'] = builds 570 r['details'] = details 571 572 # Calculate the td span for the comment and the details. 573 r["span"] = len(builderList) + 2 574 575 subs['revisions'].append(r) 576 577 # 578 # Display the footer of the page. 579 # 580 debugInfo["load_time"] = time.time() - debugInfo["load_time"] 581 return subs

582 583

584 - def content(self, request, cxt):

585 "This method builds the main console view display." 586 587 reload_time = None 588 # Check if there was an arg. Don't let people reload faster than 589 # every 15 seconds. 0 means no reload. 590 if "reload" in request.args: 591 try: 592 reload_time = int(request.args["reload"][0]) 593 if reload_time != 0: 594 reload_time = max(reload_time, 15) 595 except ValueError: 596 pass 597 598 request.setHeader('Cache-Control', 'no-cache') 599 600 # Sets the default reload time to 60 seconds. 601 if not reload_time: 602 reload_time = 60 603 604 # Append the tag to refresh the page. 605 if reload_time is not None and reload_time != 0: 606 cxt['refresh'] = reload_time 607 608 # Debug information to display at the end of the page. 609 debugInfo = cxt['debuginfo'] = dict() 610 debugInfo["load_time"] = time.time() 611 612 # get url parameters 613 # Categories to show information for. 614 categories = request.args.get("category", []) 615 # List of all builders to show on the page. 616 builders = request.args.get("builder", []) 617 # Repo used to filter the changes shown. 618 repository = request.args.get("repository", [None])[0] 619 # Project used to filter the changes shown. 620 project = request.args.get("project", [None])[0] 621 # Branch used to filter the changes shown. 622 branch = request.args.get("branch", [ANYBRANCH])[0] 623 # List of all the committers name to display on the page. 624 devName = request.args.get("name", []) 625 626 # and the data we want to render 627 status = self.getStatus(request) 628 629 # Keep only the revisions we care about. 630 # By default we process the last 40 revisions. 631 # If a dev name is passed, we look for the changes by this person in the 632 # last 80 revisions. 633 numRevs = int(request.args.get("revs", [40])[0]) 634 if devName: 635 numRevs *= 2 636 numBuilds = numRevs 637 638 # Get all changes we can find. This is a DB operation, so it must use 639 # a deferred. 640 d = self.getAllChanges(request, status, debugInfo) 641 def got_changes(allChanges): 642 debugInfo["source_all"] = len(allChanges) 643 644 revFilter = {} 645 if branch != ANYBRANCH: 646 revFilter['branch'] = branch 647 if devName: 648 revFilter['who'] = devName 649 if repository: 650 revFilter['repository'] = repository 651 if project: 652 revFilter['project'] = project 653 revisions = list(self.filterRevisions(allChanges, max_revs=numRevs, 654 filter=revFilter)) 655 debugInfo["revision_final"] = len(revisions) 656 657 # Fetch all the builds for all builders until we get the next build 658 # after lastRevision. 659 builderList = None 660 allBuilds = None 661 if revisions: 662 lastRevision = revisions[len(revisions) - 1].revision 663 debugInfo["last_revision"] = lastRevision 664 665 (builderList, allBuilds) = self.getAllBuildsForRevision(status, 666 request, 667 lastRevision, 668 numBuilds, 669 categories, 670 builders, 671 debugInfo) 672 673 debugInfo["added_blocks"] = 0 674 675 cxt.update(self.displayPage(request, status, builderList, 676 allBuilds, revisions, categories, 677 repository, project, branch, debugInfo)) 678 679 templates = request.site.buildbot_service.templates 680 template = templates.get_template("console.html") 681 data = template.render(cxt) 682 return data

683 d.addCallback(got_changes) 684 return d

Source Code for Module buildbot.status.web.console