buildbot.status.web.console

78 """Main console class. It displays a user-oriented status page. 79 Every change is a line in the page, and it shows the result of the first 80 build with this change for each slave.""" 81

82 - def __init__(self, orderByTime=False):

83 HtmlResource.__init__(self) 84 85 self.status = None 86 87 if orderByTime: 88 self.comparator = TimeRevisionComparator() 89 else: 90 self.comparator = IntegerRevisionComparator()

91

92 - def getTitle(self, request):

93 status = self.getStatus(request) 94 projectName = status.getProjectName() 95 if projectName: 96 return "BuildBot: %s" % projectName 97 else: 98 return "BuildBot"

99

100 - def getChangeManager(self, request):

101 return request.site.buildbot_service.parent.change_svc

102 103 ## 104 ## Data gathering functions 105 ## 106

107 - def getHeadBuild(self, builder):

108 """Get the most recent build for the given builder. 109 """ 110 build = builder.getBuild(-1) 111 112 # HACK: Work around #601, the head build may be None if it is 113 # locked. 114 if build is None: 115 build = builder.getBuild(-2) 116 117 return build

118

119 - def fetchChangesFromHistory(self, status, max_depth, max_builds, debugInfo):

120 """Look at the history of the builders and try to fetch as many changes 121 as possible. We need this when the main source does not contain enough 122 sourcestamps. 123 124 max_depth defines how many builds we will parse for a given builder. 125 max_builds defines how many builds total we want to parse. This is to 126 limit the amount of time we spend in this function. 127 128 This function is sub-optimal, but the information returned by this 129 function is cached, so this function won't be called more than once. 130 """ 131 132 allChanges = list() 133 build_count = 0 134 for builderName in status.getBuilderNames()[:]: 135 if build_count > max_builds: 136 break 137 138 builder = status.getBuilder(builderName) 139 build = self.getHeadBuild(builder) 140 depth = 0 141 while build and depth < max_depth and build_count < max_builds: 142 depth += 1 143 build_count += 1 144 sourcestamp = build.getSourceStamp() 145 allChanges.extend(sourcestamp.changes[:]) 146 build = build.getPreviousBuild() 147 148 debugInfo["source_fetch_len"] = len(allChanges) 149 return allChanges

150

151 - def getAllChanges(self, source, status, debugInfo):

152 """Return all the changes we can find at this time. If |source| does not 153 not have enough (less than 25), we try to fetch more from the builders 154 history.""" 155 156 g = source.eventGenerator() 157 allChanges = [] 158 while len(allChanges) < 25: 159 try: 160 c = g.next() 161 except StopIteration: 162 break 163 allChanges.append(c) 164 165 allChanges.sort(key=self.comparator.getSortingKey()) 166 167 # Remove the dups 168 prevChange = None 169 newChanges = [] 170 for change in allChanges: 171 rev = change.revision 172 if not prevChange or rev != prevChange.revision: 173 newChanges.append(change) 174 prevChange = change 175 allChanges = newChanges 176 177 return allChanges

178

179 - def getBuildDetails(self, request, builderName, build):

180 """Returns an HTML list of failures for a given build.""" 181 details = {} 182 if not build.getLogs(): 183 return details 184 185 for step in build.getSteps(): 186 (result, reason) = step.getResults() 187 if result == builder.FAILURE: 188 name = step.getName() 189 190 # Remove html tags from the error text. 191 stripHtml = re.compile(r'<.*?>') 192 strippedDetails = stripHtml.sub('', ' '.join(step.getText())) 193 194 details['buildername'] = builderName 195 details['status'] = strippedDetails 196 details['reason'] = reason 197 logs = details['logs'] = [] 198 199 if step.getLogs(): 200 for log in step.getLogs(): 201 logname = log.getName() 202 logurl = request.childLink( 203 "../builders/%s/builds/%s/steps/%s/logs/%s" % 204 (urllib.quote(builderName), 205 build.getNumber(), 206 urllib.quote(name), 207 urllib.quote(logname))) 208 logs.append(dict(url=logurl, name=logname)) 209 return details

210

211 - def getBuildsForRevision(self, request, builder, builderName, lastRevision, 212 numBuilds, debugInfo):

213 """Return the list of all the builds for a given builder that we will 214 need to be able to display the console page. We start by the most recent 215 build, and we go down until we find a build that was built prior to the 216 last change we are interested in.""" 217 218 revision = lastRevision 219 220 builds = [] 221 build = self.getHeadBuild(builder) 222 number = 0 223 while build and number < numBuilds: 224 debugInfo["builds_scanned"] += 1 225 number += 1 226 227 # Get the last revision in this build. 228 # We first try "got_revision", but if it does not work, then 229 # we try "revision". 230 got_rev = -1 231 try: 232 got_rev = build.getProperty("got_revision") 233 if not self.comparator.isValidRevision(got_rev): 234 got_rev = -1 235 except KeyError: 236 pass 237 238 try: 239 if got_rev == -1: 240 got_rev = build.getProperty("revision") 241 if not self.comparator.isValidRevision(got_rev): 242 got_rev = -1 243 except: 244 pass 245 246 # We ignore all builds that don't have last revisions. 247 # TODO(nsylvain): If the build is over, maybe it was a problem 248 # with the update source step. We need to find a way to tell the 249 # user that his change might have broken the source update. 250 if got_rev and got_rev != -1: 251 details = self.getBuildDetails(request, builderName, build) 252 devBuild = DevBuild(got_rev, build, details) 253 builds.append(devBuild) 254 255 # Now break if we have enough builds. 256 current_revision = self.getChangeForBuild( 257 build, revision) 258 if self.comparator.isRevisionEarlier( 259 devBuild, current_revision): 260 break 261 262 build = build.getPreviousBuild() 263 264 return builds

265

266 - def getChangeForBuild(self, build, revision):

267 if not build or not build.getChanges(): # Forced build 268 return DevBuild(revision, build, None) 269 270 for change in build.getChanges(): 271 if change.revision == revision: 272 return change 273 274 # No matching change, return the last change in build. 275 changes = list(build.getChanges()) 276 changes.sort(key=self.comparator.getSortingKey()) 277 return changes[-1]

278

279 - def getAllBuildsForRevision(self, status, request, lastRevision, numBuilds, 280 categories, builders, debugInfo):

281 """Returns a dictionary of builds we need to inspect to be able to 282 display the console page. The key is the builder name, and the value is 283 an array of build we care about. We also returns a dictionary of 284 builders we care about. The key is it's category. 285 286 lastRevision is the last revision we want to display in the page. 287 categories is a list of categories to display. It is coming from the 288 HTTP GET parameters. 289 builders is a list of builders to display. It is coming from the HTTP 290 GET parameters. 291 """ 292 293 allBuilds = dict() 294 295 # List of all builders in the dictionary. 296 builderList = dict() 297 298 debugInfo["builds_scanned"] = 0 299 # Get all the builders. 300 builderNames = status.getBuilderNames()[:] 301 for builderName in builderNames: 302 builder = status.getBuilder(builderName) 303 304 # Make sure we are interested in this builder. 305 if categories and builder.category not in categories: 306 continue 307 if builders and builderName not in builders: 308 continue 309 310 # We want to display this builder. 311 category = builder.category or "default" 312 # Strip the category to keep only the text before the first |. 313 # This is a hack to support the chromium usecase where they have 314 # multiple categories for each slave. We use only the first one. 315 # TODO(nsylvain): Create another way to specify "display category" 316 # in master.cfg. 317 category = category.split('|')[0] 318 if not builderList.get(category): 319 builderList[category] = [] 320 321 # Append this builder to the dictionary of builders. 322 builderList[category].append(builderName) 323 # Set the list of builds for this builder. 324 allBuilds[builderName] = self.getBuildsForRevision(request, 325 builder, 326 builderName, 327 lastRevision, 328 numBuilds, 329 debugInfo) 330 331 return (builderList, allBuilds)

332 333 334 ## 335 ## Display functions 336 ## 337

338 - def displayCategories(self, builderList, debugInfo):

339 """Display the top category line.""" 340 341 count = 0 342 for category in builderList: 343 count += len(builderList[category]) 344 345 categories = builderList.keys() 346 categories.sort() 347 348 cs = [] 349 350 for category in categories: 351 c = {} 352 # TODO(nsylvain): Another hack to display the category in a pretty 353 # way. If the master owner wants to display the categories in a 354 # given order, he/she can prepend a number to it. This number won't 355 # be shown. 356 c["name"] = category.lstrip('0123456789') 357 358 # To be able to align the table correctly, we need to know 359 # what percentage of space this category will be taking. This is 360 # (#Builders in Category) / (#Builders Total) * 100. 361 c["size"] = (len(builderList[category]) * 100) / count 362 cs.append(c) 363 364 return cs

365

366 - def displaySlaveLine(self, status, builderList, debugInfo):

367 """Display a line the shows the current status for all the builders we 368 care about.""" 369 370 nbSlaves = 0 371 372 # Get the number of builders. 373 for category in builderList: 374 nbSlaves += len(builderList[category]) 375 376 # Get the categories, and order them alphabetically. 377 categories = builderList.keys() 378 categories.sort() 379 380 slaves = {} 381 382 # For each category, we display each builder. 383 for category in categories: 384 slaves[category] = [] 385 # For each builder in this category, we set the build info and we 386 # display the box. 387 for builder in builderList[category]: 388 s = {} 389 s["color"] = "notstarted" 390 s["title"] = builder 391 s["url"] = "./builders/%s" % urllib.quote(builder) 392 state, builds = status.getBuilder(builder).getState() 393 # Check if it's offline, if so, the box is purple. 394 if state == "offline": 395 s["color"] = "offline" 396 else: 397 # If not offline, then display the result of the last 398 # finished build. 399 build = self.getHeadBuild(status.getBuilder(builder)) 400 while build and not build.isFinished(): 401 build = build.getPreviousBuild() 402 403 if build: 404 s["color"] = getResultsClass(build.getResults(), None, 405 False) 406 407 slaves[category].append(s) 408 409 return slaves

410

411 - def displayStatusLine(self, builderList, allBuilds, revision, debugInfo):

412 """Display the boxes that represent the status of each builder in the 413 first build "revision" was in. Returns an HTML list of errors that 414 happened during these builds.""" 415 416 details = [] 417 nbSlaves = 0 418 for category in builderList: 419 nbSlaves += len(builderList[category]) 420 421 # Sort the categories. 422 categories = builderList.keys() 423 categories.sort() 424 425 builds = {} 426 427 # Display the boxes by category group. 428 for category in categories: 429 430 builds[category] = [] 431 432 # Display the boxes for each builder in this category. 433 for builder in builderList[category]: 434 introducedIn = None 435 firstNotIn = None 436 437 # Find the first build that does not include the revision. 438 for build in allBuilds[builder]: 439 if self.comparator.isRevisionEarlier(build, revision): 440 firstNotIn = build 441 break 442 else: 443 introducedIn = build 444 445 # Get the results of the first build with the revision, and the 446 # first build that does not include the revision. 447 results = None 448 previousResults = None 449 if introducedIn: 450 results = introducedIn.results 451 if firstNotIn: 452 previousResults = firstNotIn.results 453 454 isRunning = False 455 if introducedIn and not introducedIn.isFinished: 456 isRunning = True 457 458 url = "./waterfall" 459 title = builder 460 tag = "" 461 current_details = {} 462 if introducedIn: 463 current_details = introducedIn.details or "" 464 url = "./buildstatus?builder=%s&number=%s" % (urllib.quote(builder), 465 introducedIn.number) 466 title += " " 467 title += urllib.quote(' '.join(introducedIn.text), ' \n\\/:') 468 469 builderStrip = builder.replace(' ', '') 470 builderStrip = builderStrip.replace('(', '') 471 builderStrip = builderStrip.replace(')', '') 472 builderStrip = builderStrip.replace('.', '') 473 tag = "Tag%s%s" % (builderStrip, introducedIn.number) 474 475 if isRunning: 476 title += ' ETA: %ds' % (introducedIn.eta or 0) 477 478 resultsClass = getResultsClass(results, previousResults, isRunning) 479 480 b = {} 481 b["url"] = url 482 b["title"] = title 483 b["color"] = resultsClass 484 b["tag"] = tag 485 486 builds[category].append(b) 487 488 # If the box is red, we add the explaination in the details 489 # section. 490 if current_details and resultsClass == "failure": 491 details.append(current_details) 492 493 return (builds, details)

494

495 - def filterRevisions(self, revisions, filter=None, max_revs=None):

496 """Filter a set of revisions based on any number of filter criteria. 497 If specified, filter should be a dict with keys corresponding to 498 revision attributes, and values of 1+ strings""" 499 if not filter: 500 if max_revs is None: 501 for rev in reversed(revisions): 502 yield DevRevision(rev) 503 else: 504 for index,rev in enumerate(reversed(revisions)): 505 if index >= max_revs: 506 break 507 yield DevRevision(rev) 508 else: 509 for index, rev in enumerate(reversed(revisions)): 510 if max_revs and index >= max_revs: 511 break 512 try: 513 for field,acceptable in filter.iteritems(): 514 if not hasattr(rev, field): 515 raise DoesNotPassFilter 516 if type(acceptable) in (str, unicode): 517 if getattr(rev, field) != acceptable: 518 raise DoesNotPassFilter 519 elif type(acceptable) in (list, tuple, set): 520 if getattr(rev, field) not in acceptable: 521 raise DoesNotPassFilter 522 yield DevRevision(rev) 523 except DoesNotPassFilter: 524 pass

525

526 - def displayPage(self, request, status, builderList, allBuilds, revisions, 527 categories, repository, branch, debugInfo):

528 """Display the console page.""" 529 # Build the main template directory with all the informations we have. 530 subs = dict() 531 subs["branch"] = branch or 'trunk' 532 subs["repository"] = repository 533 if categories: 534 subs["categories"] = ' '.join(categories) 535 subs["time"] = time.strftime("%a %d %b %Y %H:%M:%S", 536 time.localtime(util.now())) 537 subs["debugInfo"] = debugInfo 538 subs["ANYBRANCH"] = ANYBRANCH 539 540 if builderList: 541 subs["categories"] = self.displayCategories(builderList, debugInfo) 542 subs['slaves'] = self.displaySlaveLine(status, builderList, debugInfo) 543 else: 544 subs["categories"] = [] 545 546 subs['revisions'] = [] 547 548 # For each revision we show one line 549 for revision in revisions: 550 r = {} 551 552 # Fill the dictionary with this new information 553 r['id'] = revision.revision 554 r['link'] = revision.revlink 555 r['who'] = revision.who 556 r['date'] = revision.date 557 r['comments'] = revision.comments 558 r['repository'] = revision.repository 559 r['project'] = revision.project 560 561 # Display the status for all builders. 562 (builds, details) = self.displayStatusLine(builderList, 563 allBuilds, 564 revision, 565 debugInfo) 566 r['builds'] = builds 567 r['details'] = details 568 569 # Calculate the td span for the comment and the details. 570 r["span"] = len(builderList) + 2 571 572 subs['revisions'].append(r) 573 574 # 575 # Display the footer of the page. 576 # 577 debugInfo["load_time"] = time.time() - debugInfo["load_time"] 578 return subs

579 580

581 - def content(self, request, cxt):

582 "This method builds the main console view display." 583 584 reload_time = None 585 # Check if there was an arg. Don't let people reload faster than 586 # every 15 seconds. 0 means no reload. 587 if "reload" in request.args: 588 try: 589 reload_time = int(request.args["reload"][0]) 590 if reload_time != 0: 591 reload_time = max(reload_time, 15) 592 except ValueError: 593 pass 594 595 request.setHeader('Cache-Control', 'no-cache') 596 597 # Sets the default reload time to 60 seconds. 598 if not reload_time: 599 reload_time = 60 600 601 # Append the tag to refresh the page. 602 if reload_time is not None and reload_time != 0: 603 cxt['refresh'] = reload_time 604 605 # Debug information to display at the end of the page. 606 debugInfo = cxt['debuginfo'] = dict() 607 debugInfo["load_time"] = time.time() 608 609 # get url parameters 610 # Categories to show information for. 611 categories = request.args.get("category", []) 612 # List of all builders to show on the page. 613 builders = request.args.get("builder", []) 614 # Repo used to filter the changes shown. 615 repository = request.args.get("repository", [None])[0] 616 # Branch used to filter the changes shown. 617 branch = request.args.get("branch", [ANYBRANCH])[0] 618 # List of all the committers name to display on the page. 619 devName = request.args.get("name", []) 620 621 # and the data we want to render 622 status = self.getStatus(request) 623 624 # Get all revisions we can find. 625 source = self.getChangeManager(request) 626 allChanges = self.getAllChanges(source, status, debugInfo) 627 628 debugInfo["source_all"] = len(allChanges) 629 630 # Keep only the revisions we care about. 631 # By default we process the last 40 revisions. 632 # If a dev name is passed, we look for the changes by this person in the 633 # last 80 revisions. 634 numRevs = int(request.args.get("revs", [40])[0]) 635 if devName: 636 numRevs *= 2 637 numBuilds = numRevs 638 639 revFilter = {} 640 if branch != ANYBRANCH: 641 revFilter['branch'] = branch 642 if devName: 643 revFilter['who'] = devName 644 if repository: 645 revFilter['repository'] = repository 646 revisions = list(self.filterRevisions(allChanges, max_revs=numRevs, filter=revFilter)) 647 debugInfo["revision_final"] = len(revisions) 648 649 # Fetch all the builds for all builders until we get the next build 650 # after lastRevision. 651 builderList = None 652 allBuilds = None 653 if revisions: 654 lastRevision = revisions[len(revisions) - 1].revision 655 debugInfo["last_revision"] = lastRevision 656 657 (builderList, allBuilds) = self.getAllBuildsForRevision(status, 658 request, 659 lastRevision, 660 numBuilds, 661 categories, 662 builders, 663 debugInfo) 664 665 debugInfo["added_blocks"] = 0 666 667 cxt.update(self.displayPage(request, status, builderList, allBuilds, 668 revisions, categories, repository, branch, debugInfo)) 669 670 template = request.site.buildbot_service.templates.get_template("console.html") 671 data = template.render(cxt) 672 return data

Source Code for Module buildbot.status.web.console