buildbot.status.web.console

73 """Main console class. It displays a user-oriented status page. 74 Every change is a line in the page, and it shows the result of the first 75 build with this change for each slave.""" 76

77 - def __init__(self, allowForce=True, css=None, orderByTime=False):

78 HtmlResource.__init__(self) 79 80 self.status = None 81 self.control = None 82 self.changemaster = None 83 self.initialRevs = None 84 85 self.allowForce = allowForce 86 self.css = css 87 88 if orderByTime: 89 self.comparator = TimeRevisionComparator() 90 else: 91 self.comparator = IntegerRevisionComparator()

92

93 - def getTitle(self, request):

94 status = self.getStatus(request) 95 projectName = status.getProjectName() 96 if projectName: 97 return "BuildBot: %s" % projectName 98 else: 99 return "BuildBot"

100

101 - def getChangemaster(self, request):

102 return request.site.buildbot_service.parent.change_svc

103

104 - def head(self, request):

105 # Start by adding all the javascript functions we have. 106 head = "<script type='text/javascript'> %s </script>" % js.JAVASCRIPT 107 108 reload_time = None 109 # Check if there was an arg. Don't let people reload faster than 110 # every 15 seconds. 0 means no reload. 111 if "reload" in request.args: 112 try: 113 reload_time = int(request.args["reload"][0]) 114 if reload_time != 0: 115 reload_time = max(reload_time, 15) 116 except ValueError: 117 pass 118 119 # Sets the default reload time to 60 seconds. 120 if not reload_time: 121 reload_time = 60 122 123 # Append the tag to refresh the page. 124 if reload_time is not None and reload_time != 0: 125 head += '<meta http-equiv="refresh" content="%d">\n' % reload_time 126 return head

127 128 129 ## 130 ## Data gathering functions 131 ## 132

133 - def getHeadBuild(self, builder):

134 """Get the most recent build for the given builder. 135 """ 136 build = builder.getBuild(-1) 137 138 # HACK: Work around #601, the head build may be None if it is 139 # locked. 140 if build is None: 141 build = builder.getBuild(-2) 142 143 return build

144

145 - def fetchChangesFromHistory(self, status, max_depth, max_builds, debugInfo):

146 """Look at the history of the builders and try to fetch as many changes 147 as possible. We need this when the main source does not contain enough 148 sourcestamps. 149 150 max_depth defines how many builds we will parse for a given builder. 151 max_builds defines how many builds total we want to parse. This is to 152 limit the amount of time we spend in this function. 153 154 This function is sub-optimal, but the information returned by this 155 function is cached, so this function won't be called more than once. 156 """ 157 158 allChanges = list() 159 build_count = 0 160 for builderName in status.getBuilderNames()[:]: 161 if build_count > max_builds: 162 break 163 164 builder = status.getBuilder(builderName) 165 build = self.getHeadBuild(builder) 166 depth = 0 167 while build and depth < max_depth and build_count < max_builds: 168 depth += 1 169 build_count += 1 170 sourcestamp = build.getSourceStamp() 171 allChanges.extend(sourcestamp.changes[:]) 172 build = build.getPreviousBuild() 173 174 debugInfo["source_fetch_len"] = len(allChanges) 175 return allChanges

176

177 - def getAllChanges(self, source, status, debugInfo):

178 """Return all the changes we can find at this time. If |source| does not 179 not have enough (less than 25), we try to fetch more from the builders 180 history.""" 181 182 allChanges = list() 183 allChanges.extend(source.changes[:]) 184 185 debugInfo["source_len"] = len(source.changes) 186 187 if len(allChanges) < 25: 188 # There is not enough revisions in the source.changes. It happens 189 # quite a lot because buildbot mysteriously forget about changes 190 # once in a while during restart. 191 # Let's try to get more changes from the builders. 192 # We check the last 10 builds of all builders, and stop when we 193 # are done, or have looked at 100 builds. 194 # We do this only once! 195 if not self.initialRevs: 196 self.initialRevs = self.fetchChangesFromHistory(status, 10, 100, 197 debugInfo) 198 199 allChanges.extend(self.initialRevs) 200 201 # the new changes are not sorted, and can contain duplicates. 202 # Sort the list. 203 allChanges.sort(lambda a, b: cmp(getattr(a, self.comparator.getSortingKey()), getattr(b, self.comparator.getSortingKey()))) 204 205 # Remove the dups 206 prevChange = None 207 newChanges = [] 208 for change in allChanges: 209 rev = change.revision 210 if not prevChange or rev != prevChange.revision: 211 newChanges.append(change) 212 prevChange = change 213 allChanges = newChanges 214 215 return allChanges

216

217 - def stripRevisions(self, allChanges, numRevs, branch, devName):

218 """Returns a subset of changesn from allChanges that matches the query. 219 220 allChanges is the list of all changes we know about. 221 numRevs is the number of changes we will inspect from allChanges. We 222 do not want to inspect all of them or it would be too slow. 223 branch is the branch we are interested in. Changes not in this branch 224 will be ignored. 225 devName is the developper name. Changes have not been submitted by this 226 person will be ignored. 227 """ 228 229 revisions = [] 230 231 if not allChanges: 232 return revisions 233 234 totalRevs = len(allChanges) 235 for i in range(totalRevs-1, totalRevs-numRevs, -1): 236 if i < 0: 237 break 238 change = allChanges[i] 239 if branch == ANYBRANCH or branch == change.branch: 240 if not devName or change.who in devName: 241 242 rev = DevRevision(change.revision, change.who, 243 change.comments, change.getTime(), 244 getattr(change, 'revlink', None), 245 change.when) 246 revisions.append(rev) 247 248 return revisions

249

250 - def getBuildDetails(self, request, builderName, build):

251 """Returns an HTML list of failures for a given build.""" 252 details = "" 253 if build.getLogs(): 254 for step in build.getSteps(): 255 (result, reason) = step.getResults() 256 if result == builder.FAILURE: 257 name = step.getName() 258 259 # Remove html tags from the error text. 260 stripHtml = re.compile(r'<.*?>') 261 strippedDetails = stripHtml .sub('', ' '.join(step.getText())) 262 263 details += "<li> %s : %s. \n" % (builderName, strippedDetails) 264 if step.getLogs(): 265 details += "[ " 266 for log in step.getLogs(): 267 logname = log.getName() 268 logurl = request.childLink( 269 "../builders/%s/builds/%s/steps/%s/logs/%s" % 270 (urllib.quote(builderName), 271 build.getNumber(), 272 urllib.quote(name), 273 urllib.quote(logname))) 274 details += "<a href=\"%s\">%s</a> " % (logurl, 275 log.getName()) 276 details += "]" 277 return details

278

279 - def getBuildsForRevision(self, request, builder, builderName, lastRevision, 280 numBuilds, debugInfo):

281 """Return the list of all the builds for a given builder that we will 282 need to be able to display the console page. We start by the most recent 283 build, and we go down until we find a build that was built prior to the 284 last change we are interested in.""" 285 286 revision = lastRevision 287 288 builds = [] 289 build = self.getHeadBuild(builder) 290 number = 0 291 while build and number < numBuilds: 292 debugInfo["builds_scanned"] += 1 293 number += 1 294 295 # Get the last revision in this build. 296 # We first try "got_revision", but if it does not work, then 297 # we try "revision". 298 got_rev = -1 299 try: 300 got_rev = build.getProperty("got_revision") 301 if not self.comparator.isValidRevision(got_rev): 302 got_rev = -1 303 except KeyError: 304 pass 305 306 try: 307 if got_rev == -1: 308 got_rev = build.getProperty("revision") 309 if not self.comparator.isValidRevision(got_rev): 310 got_rev = -1 311 except: 312 pass 313 314 # We ignore all builds that don't have last revisions. 315 # TODO(nsylvain): If the build is over, maybe it was a problem 316 # with the update source step. We need to find a way to tell the 317 # user that his change might have broken the source update. 318 if got_rev and got_rev != -1: 319 details = self.getBuildDetails(request, builderName, build) 320 devBuild = DevBuild(got_rev, build.getResults(), 321 build.getNumber(), 322 build.isFinished(), 323 build.getText(), 324 build.getETA(), 325 details, 326 build.getTimes()[0]) 327 328 builds.append(devBuild) 329 330 # Now break if we have enough builds. 331 current_revision = self.getChangeForBuild( 332 builder.getBuild(-1), revision) 333 if self.comparator.isRevisionEarlier( 334 devBuild, current_revision): 335 break 336 337 build = build.getPreviousBuild() 338 339 return builds

340

341 - def getChangeForBuild(self, build, revision):

342 if not build.getChanges(): # Forced build 343 devBuild = DevBuild(revision, build.getResults(), 344 build.getNumber(), 345 build.isFinished(), 346 build.getText(), 347 build.getETA(), 348 None, 349 build.getTimes()[0]) 350 351 return devBuild 352 353 for change in build.getChanges(): 354 if change.revision == revision: 355 return change 356 357 # No matching change, return the last change in build. 358 changes = list(build.getChanges()) 359 changes.sort(lambda a, b: cmp(getattr(a, self.comparator.getSortingKey()), getattr(b, self.comparator.getSortingKey()))) 360 return changes[-1]

361

362 - def getAllBuildsForRevision(self, status, request, lastRevision, numBuilds, 363 categories, builders, debugInfo):

364 """Returns a dictionnary of builds we need to inspect to be able to 365 display the console page. The key is the builder name, and the value is 366 an array of build we care about. We also returns a dictionnary of 367 builders we care about. The key is it's category. 368 369 lastRevision is the last revision we want to display in the page. 370 categories is a list of categories to display. It is coming from the 371 HTTP GET parameters. 372 builders is a list of builders to display. It is coming from the HTTP 373 GET parameters. 374 """ 375 376 allBuilds = dict() 377 378 # List of all builders in the dictionnary. 379 builderList = dict() 380 381 debugInfo["builds_scanned"] = 0 382 # Get all the builders. 383 builderNames = status.getBuilderNames()[:] 384 for builderName in builderNames: 385 builder = status.getBuilder(builderName) 386 387 # Make sure we are interested in this builder. 388 if categories and builder.category not in categories: 389 continue 390 if builders and builderName not in builders: 391 continue 392 393 # We want to display this builder. 394 category = builder.category or "default" 395 # Strip the category to keep only the text before the first |. 396 # This is a hack to support the chromium usecase where they have 397 # multiple categories for each slave. We use only the first one. 398 # TODO(nsylvain): Create another way to specify "display category" 399 # in master.cfg. 400 category = category.split('|')[0] 401 if not builderList.get(category): 402 builderList[category] = [] 403 404 # Append this builder to the dictionnary of builders. 405 builderList[category].append(builderName) 406 # Set the list of builds for this builder. 407 allBuilds[builderName] = self.getBuildsForRevision(request, 408 builder, 409 builderName, 410 lastRevision, 411 numBuilds, 412 debugInfo) 413 414 return (builderList, allBuilds)

415 416 417 ## 418 ## Display functions 419 ## 420

421 - def displayCategories(self, builderList, debugInfo, subs):

422 """Display the top category line.""" 423 424 data = res.main_line_category_header.substitute(subs) 425 count = 0 426 for category in builderList: 427 count += len(builderList[category]) 428 429 i = 0 430 categories = builderList.keys() 431 categories.sort() 432 for category in categories: 433 # First, we add a flag to say if it's the first or the last one. 434 # This is useful is your css is doing rounding at the edge of the 435 # tables. 436 subs["first"] = "" 437 subs["last"] = "" 438 if i == 0: 439 subs["first"] = "first" 440 if i == len(builderList) -1: 441 subs["last"] = "last" 442 443 # TODO(nsylvain): Another hack to display the category in a pretty 444 # way. If the master owner wants to display the categories in a 445 # given order, he/she can prepend a number to it. This number won't 446 # be shown. 447 subs["category"] = category.lstrip('0123456789') 448 449 # To be able to align the table correctly, we need to know 450 # what percentage of space this category will be taking. This is 451 # (#Builders in Category) / (#Builders Total) * 100. 452 subs["size"] = (len(builderList[category]) * 100) / count 453 data += res.main_line_category_name.substitute(subs) 454 i += 1 455 data += res.main_line_category_footer.substitute(subs) 456 return data

457

458 - def displaySlaveLine(self, status, builderList, debugInfo, subs):

459 """Display a line the shows the current status for all the builders we 460 care about.""" 461 462 data = "" 463 464 # Display the first TD (empty) element. 465 subs["last"] = "" 466 if len(builderList) == 1: 467 subs["last"] = "last" 468 data += res.main_line_slave_header.substitute(subs) 469 470 nbSlaves = 0 471 subs["first"] = "" 472 473 # Get the number of builders. 474 for category in builderList: 475 nbSlaves += len(builderList[category]) 476 477 i = 0 478 479 # Get the catefories, and order them alphabetically. 480 categories = builderList.keys() 481 categories.sort() 482 483 # For each category, we display each builder. 484 for category in categories: 485 subs["last"] = "" 486 487 # If it's the last category, we set the "last" flag. 488 if i == len(builderList) - 1: 489 subs["last"] = "last" 490 491 # This is not the first category, we need to add the spacing we have 492 # between 2 categories. 493 if i != 0: 494 data += res.main_line_slave_section.substitute(subs) 495 496 i += 1 497 498 # For each builder in this category, we set the build info and we 499 # display the box. 500 for builder in builderList[category]: 501 subs["color"] = "notstarted" 502 subs["title"] = builder 503 subs["url"] = "./builders/%s" % urllib.quote(builder) 504 state, builds = status.getBuilder(builder).getState() 505 # Check if it's offline, if so, the box is purple. 506 if state == "offline": 507 subs["color"] = "exception" 508 else: 509 # If not offline, then display the result of the last 510 # finished build. 511 build = self.getHeadBuild(status.getBuilder(builder)) 512 while build and not build.isFinished(): 513 build = build.getPreviousBuild() 514 515 if build: 516 subs["color"] = getResultsClass(build.getResults(), None, 517 False) 518 519 data += res.main_line_slave_status.substitute(subs) 520 521 data += res.main_line_slave_footer.substitute(subs) 522 return data

523

524 - def displayStatusLine(self, builderList, allBuilds, revision, debugInfo, 525 subs):

526 """Display the boxes that represent the status of each builder in the 527 first build "revision" was in. Returns an HTML list of errors that 528 happened during these builds.""" 529 530 data = "" 531 532 # Display the first TD (empty) element. 533 subs["last"] = "" 534 if len(builderList) == 1: 535 subs["last"] = "last" 536 data += res.main_line_status_header.substitute(subs) 537 538 details = "" 539 nbSlaves = 0 540 subs["first"] = "" 541 for category in builderList: 542 nbSlaves += len(builderList[category]) 543 544 i = 0 545 # Sort the categories. 546 categories = builderList.keys() 547 categories.sort() 548 549 # Display the boxes by category group. 550 for category in categories: 551 # Last category? We set the "last" flag. 552 subs["last"] = "" 553 if i == len(builderList) - 1: 554 subs["last"] = "last" 555 556 # Not the first category? We add the spacing between 2 categories. 557 if i != 0: 558 data += res.main_line_status_section.substitute(subs) 559 i += 1 560 561 # Display the boxes for each builder in this category. 562 for builder in builderList[category]: 563 introducedIn = None 564 firstNotIn = None 565 566 # Find the first build that does not include the revision. 567 for build in allBuilds[builder]: 568 if self.comparator.isRevisionEarlier(build, revision): 569 firstNotIn = build 570 break 571 else: 572 introducedIn = build 573 574 # Get the results of the first build with the revision, and the 575 # first build that does not include the revision. 576 results = None 577 previousResults = None 578 if introducedIn: 579 results = introducedIn.results 580 if firstNotIn: 581 previousResults = firstNotIn.results 582 583 isRunning = False 584 if introducedIn and not introducedIn.isFinished: 585 isRunning = True 586 587 url = "./waterfall" 588 title = builder 589 tag = "" 590 current_details = None 591 if introducedIn: 592 current_details = introducedIn.details or "" 593 url = "./buildstatus?builder=%s&number=%s" % (urllib.quote(builder), 594 introducedIn.number) 595 title += " " 596 title += urllib.quote(' '.join(introducedIn.text), ' \n\\/:') 597 598 builderStrip = builder.replace(' ', '') 599 builderStrip = builderStrip.replace('(', '') 600 builderStrip = builderStrip.replace(')', '') 601 builderStrip = builderStrip.replace('.', '') 602 tag = "Tag%s%s" % (builderStrip, introducedIn.number) 603 604 if isRunning: 605 title += ' ETA: %ds' % (introducedIn.eta or 0) 606 607 resultsClass = getResultsClass(results, previousResults, isRunning) 608 subs["url"] = url 609 subs["title"] = title 610 subs["color"] = resultsClass 611 subs["tag"] = tag 612 613 data += res.main_line_status_box.substitute(subs) 614 615 # If the box is red, we add the explaination in the details 616 # section. 617 if current_details and resultsClass == "failure": 618 details += current_details 619 620 data += res.main_line_status_footer.substitute(subs) 621 return (data, details)

622

623 - def displayPage(self, request, status, builderList, allBuilds, revisions, 624 categories, branch, debugInfo):

625 """Display the console page.""" 626 # Build the main template directory with all the informations we have. 627 subs = dict() 628 subs["projectUrl"] = status.getProjectURL() or "" 629 subs["projectName"] = status.getProjectName() or "" 630 subs["branch"] = branch or 'trunk' 631 if categories: 632 subs["categories"] = ' '.join(categories) 633 subs["welcomeUrl"] = self.path_to_root(request) + "index.html" 634 subs["version"] = version 635 subs["time"] = time.strftime("%a %d %b %Y %H:%M:%S", 636 time.localtime(util.now())) 637 subs["debugInfo"] = debugInfo 638 639 640 # 641 # Show the header. 642 # 643 644 data = res.top_header.substitute(subs) 645 data += res.top_info_name.substitute(subs) 646 647 if categories: 648 data += res.top_info_categories.substitute(subs) 649 650 if branch != ANYBRANCH: 651 data += res.top_info_branch.substitute(subs) 652 653 data += res.top_info_name_end.substitute(subs) 654 # Display the legend. 655 data += res.top_legend.substitute(subs) 656 657 # Display the personalize box. 658 data += res.top_personalize.substitute(subs) 659 660 data += res.top_footer.substitute(subs) 661 662 663 # 664 # Display the main page 665 # 666 data += res.main_header.substitute(subs) 667 668 # "Alt" is set for every other line, to be able to switch the background 669 # color. 670 subs["alt"] = "Alt" 671 subs["first"] = "" 672 subs["last"] = "" 673 674 # Display the categories if there is more than 1. 675 if builderList and len(builderList) > 1: 676 dataToAdd = self.displayCategories(builderList, debugInfo, subs) 677 data += dataToAdd 678 679 # Display the build slaves status. 680 if builderList: 681 dataToAdd = self.displaySlaveLine(status, builderList, debugInfo, 682 subs) 683 data += dataToAdd 684 685 # For each revision we show one line 686 for revision in revisions: 687 if not subs["alt"]: 688 subs["alt"] = "Alt" 689 else: 690 subs["alt"] = "" 691 692 # Fill the dictionnary with these new information 693 subs["revision"] = revision.revision 694 if revision.revlink: 695 subs["revision_link"] = ("<a href=\"%s\">%s</a>" 696 % (revision.revlink, 697 revision.revision)) 698 else: 699 subs["revision_link"] = revision.revision 700 subs["who"] = revision.who 701 subs["date"] = revision.date 702 comment = revision.comments or "" 703 subs["comments"] = comment.replace('<', '<').replace('>', '>') 704 comment_quoted = urllib.quote(subs["comments"].encode("utf-8")) 705 706 # Display the revision number and the committer. 707 data += res.main_line_info.substitute(subs) 708 709 # Display the status for all builders. 710 (dataToAdd, details) = self.displayStatusLine(builderList, 711 allBuilds, 712 revision, 713 debugInfo, 714 subs) 715 data += dataToAdd 716 717 # Calculate the td span for the comment and the details. 718 subs["span"] = len(builderList) + 2 719 720 # Display the details of the failures, if any. 721 if details: 722 subs["details"] = details 723 data += res.main_line_details.substitute(subs) 724 725 # Display the comments for this revision 726 data += res.main_line_comments.substitute(subs) 727 728 data += res.main_footer.substitute(subs) 729 730 # 731 # Display the footer of the page. 732 # 733 debugInfo["load_time"] = time.time() - debugInfo["load_time"] 734 data += res.bottom.substitute(subs) 735 return data

736

737 - def body(self, request):

738 "This method builds the main console view display." 739 740 # Debug information to display at the end of the page. 741 debugInfo = dict() 742 debugInfo["load_time"] = time.time() 743 744 # get url parameters 745 # Categories to show information for. 746 categories = request.args.get("category", []) 747 # List of all builders to show on the page. 748 builders = request.args.get("builder", []) 749 # Branch used to filter the changes shown. 750 branch = request.args.get("branch", [ANYBRANCH])[0] 751 # List of all the committers name to display on the page. 752 devName = request.args.get("name", []) 753 754 # and the data we want to render 755 status = self.getStatus(request) 756 757 projectURL = status.getProjectURL() 758 projectName = status.getProjectName() 759 760 # Get all revisions we can find. 761 source = self.getChangemaster(request) 762 allChanges = self.getAllChanges(source, status, debugInfo) 763 764 debugInfo["source_all"] = len(allChanges) 765 766 # Keep only the revisions we care about. 767 # By default we process the last 40 revisions. 768 # If a dev name is passed, we look for the changes by this person in the 769 # last 80 revisions. 770 numRevs = 40 771 if devName: 772 numRevs *= 2 773 numBuilds = numRevs 774 775 776 revisions = self.stripRevisions(allChanges, numRevs, branch, devName) 777 debugInfo["revision_final"] = len(revisions) 778 779 # Fetch all the builds for all builders until we get the next build 780 # after lastRevision. 781 builderList = None 782 allBuilds = None 783 if revisions: 784 lastRevision = revisions[len(revisions)-1].revision 785 debugInfo["last_revision"] = lastRevision 786 787 (builderList, allBuilds) = self.getAllBuildsForRevision(status, 788 request, 789 lastRevision, 790 numBuilds, 791 categories, 792 builders, 793 debugInfo) 794 795 debugInfo["added_blocks"] = 0 796 797 data = "" 798 data += self.displayPage(request, status, builderList, allBuilds, 799 revisions, categories, branch, debugInfo) 800 801 return data

Source Code for Module buildbot.status.web.console