buildbot.status.web.console

93 """Main console class. It displays a user-oriented status page. 94 Every change is a line in the page, and it shows the result of the first 95 build with this change for each slave.""" 96

97 - def __init__(self, orderByTime=False):

98 HtmlResource.__init__(self) 99 100 self.status = None 101 102 if orderByTime: 103 self.comparator = TimeRevisionComparator() 104 else: 105 self.comparator = IntegerRevisionComparator()

106

107 - def getPageTitle(self, request):

108 status = self.getStatus(request) 109 title = status.getTitle() 110 if title: 111 return "BuildBot: %s" % title 112 else: 113 return "BuildBot"

114

115 - def getChangeManager(self, request):

116 return request.site.buildbot_service.parent.change_svc

117 118 ## 119 ## Data gathering functions 120 ## 121

122 - def getHeadBuild(self, builder):

123 """Get the most recent build for the given builder. 124 """ 125 build = builder.getBuild(-1) 126 127 # HACK: Work around #601, the head build may be None if it is 128 # locked. 129 if build is None: 130 build = builder.getBuild(-2) 131 132 return build

133

134 - def fetchChangesFromHistory(self, status, max_depth, max_builds, debugInfo):

135 """Look at the history of the builders and try to fetch as many changes 136 as possible. We need this when the main source does not contain enough 137 sourcestamps. 138 139 max_depth defines how many builds we will parse for a given builder. 140 max_builds defines how many builds total we want to parse. This is to 141 limit the amount of time we spend in this function. 142 143 This function is sub-optimal, but the information returned by this 144 function is cached, so this function won't be called more than once. 145 """ 146 147 allChanges = list() 148 build_count = 0 149 for builderName in status.getBuilderNames()[:]: 150 if build_count > max_builds: 151 break 152 153 builder = status.getBuilder(builderName) 154 build = self.getHeadBuild(builder) 155 depth = 0 156 while build and depth < max_depth and build_count < max_builds: 157 depth += 1 158 build_count += 1 159 sourcestamp = build.getSourceStamps()[0] 160 allChanges.extend(sourcestamp.changes[:]) 161 build = build.getPreviousBuild() 162 163 debugInfo["source_fetch_len"] = len(allChanges) 164 return allChanges

165 166 @defer.inlineCallbacks

167 - def getAllChanges(self, request, status, debugInfo):

168 master = request.site.buildbot_service.master 169 170 chdicts = yield master.db.changes.getRecentChanges(25) 171 172 # convert those to Change instances 173 allChanges = yield defer.gatherResults([ 174 changes.Change.fromChdict(master, chdict) 175 for chdict in chdicts ]) 176 177 allChanges.sort(key=self.comparator.getSortingKey()) 178 179 # Remove the dups 180 prevChange = None 181 newChanges = [] 182 for change in allChanges: 183 rev = change.revision 184 if not prevChange or rev != prevChange.revision: 185 newChanges.append(change) 186 prevChange = change 187 allChanges = newChanges 188 189 defer.returnValue(allChanges)

190

191 - def getBuildDetails(self, request, builderName, build):

192 """Returns an HTML list of failures for a given build.""" 193 details = {} 194 if not build.getLogs(): 195 return details 196 197 for step in build.getSteps(): 198 (result, reason) = step.getResults() 199 if result == builder.FAILURE: 200 name = step.getName() 201 202 # Remove html tags from the error text. 203 stripHtml = re.compile(r'<.*?>') 204 strippedDetails = stripHtml.sub('', ' '.join(step.getText())) 205 206 details['buildername'] = builderName 207 details['status'] = strippedDetails 208 details['reason'] = reason 209 logs = details['logs'] = [] 210 211 if step.getLogs(): 212 for log in step.getLogs(): 213 logname = log.getName() 214 logurl = request.childLink( 215 "../builders/%s/builds/%s/steps/%s/logs/%s" % 216 (urllib.quote(builderName), 217 build.getNumber(), 218 urllib.quote(name), 219 urllib.quote(logname))) 220 logs.append(dict(url=logurl, name=logname)) 221 return details

222

223 - def getBuildsForRevision(self, request, builder, builderName, codebase, 224 lastRevision, numBuilds, debugInfo):

225 """Return the list of all the builds for a given builder that we will 226 need to be able to display the console page. We start by the most recent 227 build, and we go down until we find a build that was built prior to the 228 last change we are interested in.""" 229 230 revision = lastRevision 231 232 builds = [] 233 build = self.getHeadBuild(builder) 234 number = 0 235 while build and number < numBuilds: 236 debugInfo["builds_scanned"] += 1 237 238 got_rev = None 239 sourceStamps = build.getSourceStamps(absolute=True) 240 241 # The console page cannot handle builds that have more than 1 revision 242 if codebase is not None: 243 # Get the last revision in this build for this codebase. 244 for ss in sourceStamps: 245 if ss.codebase == codebase: 246 got_rev = ss.revision 247 break 248 elif len(sourceStamps) == 1: 249 ss = sourceStamps[0] 250 # Get the last revision in this build. 251 got_rev = ss.revision 252 253 # We ignore all builds that don't have last revisions. 254 # TODO(nsylvain): If the build is over, maybe it was a problem 255 # with the update source step. We need to find a way to tell the 256 # user that his change might have broken the source update. 257 if got_rev is not None: 258 number += 1 259 details = self.getBuildDetails(request, builderName, build) 260 devBuild = DevBuild(got_rev, build, details) 261 builds.append(devBuild) 262 263 # Now break if we have enough builds. 264 current_revision = self.getChangeForBuild( 265 build, revision) 266 if self.comparator.isRevisionEarlier( 267 devBuild, current_revision): 268 break 269 270 build = build.getPreviousBuild() 271 272 return builds

273

274 - def getChangeForBuild(self, build, revision):

275 if not build or not build.getChanges(): # Forced build 276 return DevBuild(revision, build, None) 277 278 for change in build.getChanges(): 279 if change.revision == revision: 280 return change 281 282 # No matching change, return the last change in build. 283 changes = list(build.getChanges()) 284 changes.sort(key=self.comparator.getSortingKey()) 285 return changes[-1]

286

287 - def getAllBuildsForRevision(self, status, request, codebase, lastRevision, 288 numBuilds, categories, builders, debugInfo):

289 """Returns a dictionary of builds we need to inspect to be able to 290 display the console page. The key is the builder name, and the value is 291 an array of build we care about. We also returns a dictionary of 292 builders we care about. The key is it's category. 293 294 codebase is the codebase to get revisions from 295 lastRevision is the last revision we want to display in the page. 296 categories is a list of categories to display. It is coming from the 297 HTTP GET parameters. 298 builders is a list of builders to display. It is coming from the HTTP 299 GET parameters. 300 """ 301 302 allBuilds = dict() 303 304 # List of all builders in the dictionary. 305 builderList = dict() 306 307 debugInfo["builds_scanned"] = 0 308 # Get all the builders. 309 builderNames = status.getBuilderNames()[:] 310 for builderName in builderNames: 311 builder = status.getBuilder(builderName) 312 313 # Make sure we are interested in this builder. 314 if categories and builder.category not in categories: 315 continue 316 if builders and builderName not in builders: 317 continue 318 319 # We want to display this builder. 320 category = builder.category or "default" 321 # Strip the category to keep only the text before the first |. 322 # This is a hack to support the chromium usecase where they have 323 # multiple categories for each slave. We use only the first one. 324 # TODO(nsylvain): Create another way to specify "display category" 325 # in master.cfg. 326 category = category.split('|')[0] 327 if not builderList.get(category): 328 builderList[category] = [] 329 330 # Append this builder to the dictionary of builders. 331 builderList[category].append(builderName) 332 # Set the list of builds for this builder. 333 allBuilds[builderName] = self.getBuildsForRevision(request, 334 builder, 335 builderName, 336 codebase, 337 lastRevision, 338 numBuilds, 339 debugInfo) 340 341 return (builderList, allBuilds)

342 343 344 ## 345 ## Display functions 346 ## 347

348 - def displayCategories(self, builderList, debugInfo):

349 """Display the top category line.""" 350 351 count = 0 352 for category in builderList: 353 count += len(builderList[category]) 354 355 categories = builderList.keys() 356 categories.sort() 357 358 cs = [] 359 360 for category in categories: 361 c = {} 362 363 c["name"] = category 364 365 # To be able to align the table correctly, we need to know 366 # what percentage of space this category will be taking. This is 367 # (#Builders in Category) / (#Builders Total) * 100. 368 c["size"] = (len(builderList[category]) * 100) / count 369 cs.append(c) 370 371 return cs

372

373 - def displaySlaveLine(self, status, builderList, debugInfo):

374 """Display a line the shows the current status for all the builders we 375 care about.""" 376 377 nbSlaves = 0 378 379 # Get the number of builders. 380 for category in builderList: 381 nbSlaves += len(builderList[category]) 382 383 # Get the categories, and order them alphabetically. 384 categories = builderList.keys() 385 categories.sort() 386 387 slaves = {} 388 389 # For each category, we display each builder. 390 for category in categories: 391 slaves[category] = [] 392 # For each builder in this category, we set the build info and we 393 # display the box. 394 for builder in builderList[category]: 395 s = {} 396 s["color"] = "notstarted" 397 s["pageTitle"] = builder 398 s["url"] = "./builders/%s" % urllib.quote(builder) 399 state, builds = status.getBuilder(builder).getState() 400 # Check if it's offline, if so, the box is purple. 401 if state == "offline": 402 s["color"] = "offline" 403 else: 404 # If not offline, then display the result of the last 405 # finished build. 406 build = self.getHeadBuild(status.getBuilder(builder)) 407 while build and not build.isFinished(): 408 build = build.getPreviousBuild() 409 410 if build: 411 s["color"] = getResultsClass(build.getResults(), None, 412 False) 413 414 slaves[category].append(s) 415 416 return slaves

417

418 - def displayStatusLine(self, builderList, allBuilds, revision, debugInfo):

419 """Display the boxes that represent the status of each builder in the 420 first build "revision" was in. Returns an HTML list of errors that 421 happened during these builds.""" 422 423 details = [] 424 nbSlaves = 0 425 for category in builderList: 426 nbSlaves += len(builderList[category]) 427 428 # Sort the categories. 429 categories = builderList.keys() 430 categories.sort() 431 432 builds = {} 433 434 # Display the boxes by category group. 435 for category in categories: 436 437 builds[category] = [] 438 439 # Display the boxes for each builder in this category. 440 for builder in builderList[category]: 441 introducedIn = None 442 firstNotIn = None 443 444 # Find the first build that does not include the revision. 445 for build in allBuilds[builder]: 446 if self.comparator.isRevisionEarlier(build, revision): 447 firstNotIn = build 448 break 449 else: 450 introducedIn = build 451 452 # Get the results of the first build with the revision, and the 453 # first build that does not include the revision. 454 results = None 455 previousResults = None 456 if introducedIn: 457 results = introducedIn.results 458 if firstNotIn: 459 previousResults = firstNotIn.results 460 461 isRunning = False 462 if introducedIn and not introducedIn.isFinished: 463 isRunning = True 464 465 url = "./waterfall" 466 pageTitle = builder 467 tag = "" 468 current_details = {} 469 if introducedIn: 470 current_details = introducedIn.details or "" 471 url = "./buildstatus?builder=%s&number=%s" % (urllib.quote(builder), 472 introducedIn.number) 473 pageTitle += " " 474 pageTitle += urllib.quote(' '.join(introducedIn.text), ' \n\\/:') 475 476 builderStrip = builder.replace(' ', '') 477 builderStrip = builderStrip.replace('(', '') 478 builderStrip = builderStrip.replace(')', '') 479 builderStrip = builderStrip.replace('.', '') 480 tag = "Tag%s%s" % (builderStrip, introducedIn.number) 481 482 if isRunning: 483 pageTitle += ' ETA: %ds' % (introducedIn.eta or 0) 484 485 resultsClass = getResultsClass(results, previousResults, isRunning) 486 487 b = {} 488 b["url"] = url 489 b["pageTitle"] = pageTitle 490 b["color"] = resultsClass 491 b["tag"] = tag 492 493 builds[category].append(b) 494 495 # If the box is red, we add the explaination in the details 496 # section. 497 if current_details and resultsClass == "failure": 498 details.append(current_details) 499 500 return (builds, details)

501

502 - def filterRevisions(self, revisions, filter=None, max_revs=None):

503 """Filter a set of revisions based on any number of filter criteria. 504 If specified, filter should be a dict with keys corresponding to 505 revision attributes, and values of 1+ strings""" 506 if not filter: 507 if max_revs is None: 508 for rev in reversed(revisions): 509 yield DevRevision(rev) 510 else: 511 for index,rev in enumerate(reversed(revisions)): 512 if index >= max_revs: 513 break 514 yield DevRevision(rev) 515 else: 516 for index, rev in enumerate(reversed(revisions)): 517 if max_revs and index >= max_revs: 518 break 519 try: 520 for field,acceptable in filter.iteritems(): 521 if not hasattr(rev, field): 522 raise DoesNotPassFilter 523 if type(acceptable) in (str, unicode): 524 if getattr(rev, field) != acceptable: 525 raise DoesNotPassFilter 526 elif type(acceptable) in (list, tuple, set): 527 if getattr(rev, field) not in acceptable: 528 raise DoesNotPassFilter 529 yield DevRevision(rev) 530 except DoesNotPassFilter: 531 pass

532

533 - def displayPage(self, request, status, builderList, allBuilds, codebase, 534 revisions, categories, repository, project, branch, 535 debugInfo):

536 """Display the console page.""" 537 # Build the main template directory with all the informations we have. 538 subs = dict() 539 subs["branch"] = branch or 'trunk' 540 subs["repository"] = repository 541 subs["project"] = project 542 subs["codebase"] = codebase 543 if categories: 544 subs["categories"] = ' '.join(categories) 545 subs["time"] = time.strftime("%a %d %b %Y %H:%M:%S", 546 time.localtime(util.now())) 547 subs["debugInfo"] = debugInfo 548 subs["ANYBRANCH"] = ANYBRANCH 549 550 if builderList: 551 subs["categories"] = self.displayCategories(builderList, debugInfo) 552 subs['slaves'] = self.displaySlaveLine(status, builderList, debugInfo) 553 else: 554 subs["categories"] = [] 555 556 subs['revisions'] = [] 557 558 # For each revision we show one line 559 for revision in revisions: 560 r = {} 561 562 # Fill the dictionary with this new information 563 r['id'] = revision.revision 564 r['link'] = revision.revlink 565 r['who'] = revision.who 566 r['date'] = revision.date 567 r['comments'] = revision.comments 568 r['repository'] = revision.repository 569 r['project'] = revision.project 570 571 # Display the status for all builders. 572 (builds, details) = self.displayStatusLine(builderList, 573 allBuilds, 574 revision, 575 debugInfo) 576 r['builds'] = builds 577 r['details'] = details 578 579 # Calculate the td span for the comment and the details. 580 r["span"] = len(builderList) + 2 581 582 subs['revisions'].append(r) 583 584 # 585 # Display the footer of the page. 586 # 587 debugInfo["load_time"] = time.time() - debugInfo["load_time"] 588 return subs

589 590

591 - def content(self, request, cxt):

592 "This method builds the main console view display." 593 594 reload_time = None 595 # Check if there was an arg. Don't let people reload faster than 596 # every 15 seconds. 0 means no reload. 597 if "reload" in request.args: 598 try: 599 reload_time = int(request.args["reload"][0]) 600 if reload_time != 0: 601 reload_time = max(reload_time, 15) 602 except ValueError: 603 pass 604 605 request.setHeader('Cache-Control', 'no-cache') 606 607 # Sets the default reload time to 60 seconds. 608 if not reload_time: 609 reload_time = 60 610 611 # Append the tag to refresh the page. 612 if reload_time is not None and reload_time != 0: 613 cxt['refresh'] = reload_time 614 615 # Debug information to display at the end of the page. 616 debugInfo = cxt['debuginfo'] = dict() 617 debugInfo["load_time"] = time.time() 618 619 # get url parameters 620 # Categories to show information for. 621 categories = request.args.get("category", []) 622 # List of all builders to show on the page. 623 builders = request.args.get("builder", []) 624 # Repo used to filter the changes shown. 625 repository = request.args.get("repository", [None])[0] 626 # Project used to filter the changes shown. 627 project = request.args.get("project", [None])[0] 628 # Branch used to filter the changes shown. 629 branch = request.args.get("branch", [ANYBRANCH])[0] 630 # Codebase used to filter the changes shown. 631 codebase = request.args.get("codebase", [None])[0] 632 # List of all the committers name to display on the page. 633 devName = request.args.get("name", []) 634 635 # and the data we want to render 636 status = self.getStatus(request) 637 638 # Keep only the revisions we care about. 639 # By default we process the last 40 revisions. 640 # If a dev name is passed, we look for the changes by this person in the 641 # last 80 revisions. 642 numRevs = int(request.args.get("revs", [40])[0]) 643 if devName: 644 numRevs *= 2 645 numBuilds = numRevs 646 647 # Get all changes we can find. This is a DB operation, so it must use 648 # a deferred. 649 d = self.getAllChanges(request, status, debugInfo) 650 def got_changes(allChanges): 651 debugInfo["source_all"] = len(allChanges) 652 653 revFilter = {} 654 if branch != ANYBRANCH: 655 revFilter['branch'] = branch 656 if devName: 657 revFilter['who'] = devName 658 if repository: 659 revFilter['repository'] = repository 660 if project: 661 revFilter['project'] = project 662 if codebase is not None: 663 revFilter['codebase'] = codebase 664 revisions = list(self.filterRevisions(allChanges, max_revs=numRevs, 665 filter=revFilter)) 666 debugInfo["revision_final"] = len(revisions) 667 668 # Fetch all the builds for all builders until we get the next build 669 # after lastRevision. 670 builderList = None 671 allBuilds = None 672 if revisions: 673 lastRevision = revisions[len(revisions) - 1].revision 674 debugInfo["last_revision"] = lastRevision 675 676 (builderList, allBuilds) = self.getAllBuildsForRevision(status, 677 request, 678 codebase, 679 lastRevision, 680 numBuilds, 681 categories, 682 builders, 683 debugInfo) 684 685 debugInfo["added_blocks"] = 0 686 687 cxt.update(self.displayPage(request, status, builderList, 688 allBuilds, codebase, revisions, 689 categories, repository, project, 690 branch, debugInfo)) 691 692 templates = request.site.buildbot_service.templates 693 template = templates.get_template("console.html") 694 data = template.render(cxt) 695 return data

696 d.addCallback(got_changes) 697 return d

Source Code for Module buildbot.status.web.console