Package buildbot :: Package status :: Package web :: Module console
[frames] | no frames]

Source Code for Module buildbot.status.web.console

  1  from __future__ import generators 
  2   
  3  import time 
  4  import operator 
  5  import re 
  6  import urllib 
  7   
  8  from buildbot import util 
  9  from buildbot.status import builder 
 10  from buildbot.status.web.base import HtmlResource 
 11   
12 -def getResultsClass(results, prevResults, inProgress):
13 """Given the current and past results, return the class that will be used 14 by the css to display the right color for a box.""" 15 16 if inProgress: 17 return "running" 18 19 if results is None: 20 return "notstarted" 21 22 if results == builder.SUCCESS: 23 return "success" 24 25 if results == builder.FAILURE: 26 if not prevResults: 27 # This is the bottom box. We don't know if the previous one failed 28 # or not. We assume it did not. 29 return "failure" 30 31 if prevResults != builder.FAILURE: 32 # This is a new failure. 33 return "failure" 34 else: 35 # The previous build also failed. 36 return "warnings" 37 38 # Any other results? Like EXCEPTION? 39 return "exception"
40
41 -class ANYBRANCH: pass # a flag value, used below
42
43 -class DevRevision:
44 """Helper class that contains all the information we need for a revision.""" 45
46 - def __init__(self, change):
47 self.revision = change.revision 48 self.comments = change.comments 49 self.who = change.who 50 self.date = change.getTime() 51 self.revlink = getattr(change, 'revlink', None) 52 self.when = change.when 53 self.repository = change.repository 54 self.project = change.project
55 56
57 -class DevBuild:
58 """Helper class that contains all the information we need for a build.""" 59
60 - def __init__(self, revision, build, details):
61 self.revision = revision 62 self.results = build.getResults() 63 self.number = build.getNumber() 64 self.isFinished = build.isFinished() 65 self.text = build.getText() 66 self.eta = build.getETA() 67 self.details = details 68 self.when = build.getTimes()[0] 69 self.source = build.getSourceStamp()
70 71
72 -class ConsoleStatusResource(HtmlResource):
73 """Main console class. It displays a user-oriented status page. 74 Every change is a line in the page, and it shows the result of the first 75 build with this change for each slave.""" 76
77 - def __init__(self, orderByTime=False):
78 HtmlResource.__init__(self) 79 80 self.status = None 81 82 if orderByTime: 83 self.comparator = TimeRevisionComparator() 84 else: 85 self.comparator = IntegerRevisionComparator()
86
87 - def getTitle(self, request):
88 status = self.getStatus(request) 89 projectName = status.getProjectName() 90 if projectName: 91 return "BuildBot: %s" % projectName 92 else: 93 return "BuildBot"
94
95 - def getChangeManager(self, request):
96 return request.site.buildbot_service.parent.change_svc
97 98 ## 99 ## Data gathering functions 100 ## 101
102 - def getHeadBuild(self, builder):
103 """Get the most recent build for the given builder. 104 """ 105 build = builder.getBuild(-1) 106 107 # HACK: Work around #601, the head build may be None if it is 108 # locked. 109 if build is None: 110 build = builder.getBuild(-2) 111 112 return build
113
114 - def fetchChangesFromHistory(self, status, max_depth, max_builds, debugInfo):
115 """Look at the history of the builders and try to fetch as many changes 116 as possible. We need this when the main source does not contain enough 117 sourcestamps. 118 119 max_depth defines how many builds we will parse for a given builder. 120 max_builds defines how many builds total we want to parse. This is to 121 limit the amount of time we spend in this function. 122 123 This function is sub-optimal, but the information returned by this 124 function is cached, so this function won't be called more than once. 125 """ 126 127 allChanges = list() 128 build_count = 0 129 for builderName in status.getBuilderNames()[:]: 130 if build_count > max_builds: 131 break 132 133 builder = status.getBuilder(builderName) 134 build = self.getHeadBuild(builder) 135 depth = 0 136 while build and depth < max_depth and build_count < max_builds: 137 depth += 1 138 build_count += 1 139 sourcestamp = build.getSourceStamp() 140 allChanges.extend(sourcestamp.changes[:]) 141 build = build.getPreviousBuild() 142 143 debugInfo["source_fetch_len"] = len(allChanges) 144 return allChanges
145
146 - def getAllChanges(self, source, status, debugInfo):
147 """Return all the changes we can find at this time. If |source| does not 148 not have enough (less than 25), we try to fetch more from the builders 149 history.""" 150 151 g = source.eventGenerator() 152 allChanges = [] 153 while len(allChanges) < 25: 154 try: 155 c = g.next() 156 except StopIteration: 157 break 158 allChanges.append(c) 159 160 allChanges.sort(key=self.comparator.getSortingKey()) 161 162 # Remove the dups 163 prevChange = None 164 newChanges = [] 165 for change in allChanges: 166 rev = change.revision 167 if not prevChange or rev != prevChange.revision: 168 newChanges.append(change) 169 prevChange = change 170 allChanges = newChanges 171 172 return allChanges
173
174 - def stripRevisions(self, allChanges, numRevs, branch, devName):
175 """Returns a subset of changes from allChanges that matches the query. 176 177 allChanges is the list of all changes we know about. 178 numRevs is the number of changes we will inspect from allChanges. We 179 do not want to inspect all of them or it would be too slow. 180 branch is the branch we are interested in. Changes not in this branch 181 will be ignored. 182 devName is the developper name. Changes have not been submitted by this 183 person will be ignored. 184 """ 185 186 revisions = [] 187 188 if not allChanges: 189 return revisions 190 191 totalRevs = len(allChanges) 192 for i in range(totalRevs - 1, totalRevs - numRevs, -1): 193 if i < 0: 194 break 195 change = allChanges[i] 196 if branch == ANYBRANCH or branch == change.branch: 197 if not devName or change.who in devName: 198 rev = DevRevision(change) 199 revisions.append(rev) 200 201 return revisions
202
203 - def getBuildDetails(self, request, builderName, build):
204 """Returns an HTML list of failures for a given build.""" 205 details = {} 206 if not build.getLogs(): 207 return details 208 209 for step in build.getSteps(): 210 (result, reason) = step.getResults() 211 if result == builder.FAILURE: 212 name = step.getName() 213 214 # Remove html tags from the error text. 215 stripHtml = re.compile(r'<.*?>') 216 strippedDetails = stripHtml.sub('', ' '.join(step.getText())) 217 218 details['buildername'] = builderName 219 details['status'] = strippedDetails 220 details['reason'] = reason 221 logs = details['logs'] = [] 222 223 if step.getLogs(): 224 for log in step.getLogs(): 225 logname = log.getName() 226 logurl = request.childLink( 227 "../builders/%s/builds/%s/steps/%s/logs/%s" % 228 (urllib.quote(builderName), 229 build.getNumber(), 230 urllib.quote(name), 231 urllib.quote(logname))) 232 logs.append(dict(url=logurl, name=logname)) 233 return details
234
235 - def getBuildsForRevision(self, request, builder, builderName, lastRevision, 236 numBuilds, debugInfo):
237 """Return the list of all the builds for a given builder that we will 238 need to be able to display the console page. We start by the most recent 239 build, and we go down until we find a build that was built prior to the 240 last change we are interested in.""" 241 242 revision = lastRevision 243 244 builds = [] 245 build = self.getHeadBuild(builder) 246 number = 0 247 while build and number < numBuilds: 248 debugInfo["builds_scanned"] += 1 249 number += 1 250 251 # Get the last revision in this build. 252 # We first try "got_revision", but if it does not work, then 253 # we try "revision". 254 got_rev = -1 255 try: 256 got_rev = build.getProperty("got_revision") 257 if not self.comparator.isValidRevision(got_rev): 258 got_rev = -1 259 except KeyError: 260 pass 261 262 try: 263 if got_rev == -1: 264 got_rev = build.getProperty("revision") 265 if not self.comparator.isValidRevision(got_rev): 266 got_rev = -1 267 except: 268 pass 269 270 # We ignore all builds that don't have last revisions. 271 # TODO(nsylvain): If the build is over, maybe it was a problem 272 # with the update source step. We need to find a way to tell the 273 # user that his change might have broken the source update. 274 if got_rev and got_rev != -1: 275 details = self.getBuildDetails(request, builderName, build) 276 devBuild = DevBuild(got_rev, build, details) 277 builds.append(devBuild) 278 279 # Now break if we have enough builds. 280 current_revision = self.getChangeForBuild( 281 build, revision) 282 if self.comparator.isRevisionEarlier( 283 devBuild, current_revision): 284 break 285 286 build = build.getPreviousBuild() 287 288 return builds
289
290 - def getChangeForBuild(self, build, revision):
291 if not build or not build.getChanges(): # Forced build 292 return DevBuild(revision, build, None) 293 294 for change in build.getChanges(): 295 if change.revision == revision: 296 return change 297 298 # No matching change, return the last change in build. 299 changes = list(build.getChanges()) 300 changes.sort(key=self.comparator.getSortingKey()) 301 return changes[-1]
302
303 - def getAllBuildsForRevision(self, status, request, lastRevision, numBuilds, 304 categories, builders, debugInfo):
305 """Returns a dictionnary of builds we need to inspect to be able to 306 display the console page. The key is the builder name, and the value is 307 an array of build we care about. We also returns a dictionnary of 308 builders we care about. The key is it's category. 309 310 lastRevision is the last revision we want to display in the page. 311 categories is a list of categories to display. It is coming from the 312 HTTP GET parameters. 313 builders is a list of builders to display. It is coming from the HTTP 314 GET parameters. 315 """ 316 317 allBuilds = dict() 318 319 # List of all builders in the dictionnary. 320 builderList = dict() 321 322 debugInfo["builds_scanned"] = 0 323 # Get all the builders. 324 builderNames = status.getBuilderNames()[:] 325 for builderName in builderNames: 326 builder = status.getBuilder(builderName) 327 328 # Make sure we are interested in this builder. 329 if categories and builder.category not in categories: 330 continue 331 if builders and builderName not in builders: 332 continue 333 334 # We want to display this builder. 335 category = builder.category or "default" 336 # Strip the category to keep only the text before the first |. 337 # This is a hack to support the chromium usecase where they have 338 # multiple categories for each slave. We use only the first one. 339 # TODO(nsylvain): Create another way to specify "display category" 340 # in master.cfg. 341 category = category.split('|')[0] 342 if not builderList.get(category): 343 builderList[category] = [] 344 345 # Append this builder to the dictionnary of builders. 346 builderList[category].append(builderName) 347 # Set the list of builds for this builder. 348 allBuilds[builderName] = self.getBuildsForRevision(request, 349 builder, 350 builderName, 351 lastRevision, 352 numBuilds, 353 debugInfo) 354 355 return (builderList, allBuilds)
356 357 358 ## 359 ## Display functions 360 ## 361
362 - def displayCategories(self, builderList, debugInfo):
363 """Display the top category line.""" 364 365 count = 0 366 for category in builderList: 367 count += len(builderList[category]) 368 369 categories = builderList.keys() 370 categories.sort() 371 372 cs = [] 373 374 for category in categories: 375 c = {} 376 # TODO(nsylvain): Another hack to display the category in a pretty 377 # way. If the master owner wants to display the categories in a 378 # given order, he/she can prepend a number to it. This number won't 379 # be shown. 380 c["name"] = category.lstrip('0123456789') 381 382 # To be able to align the table correctly, we need to know 383 # what percentage of space this category will be taking. This is 384 # (#Builders in Category) / (#Builders Total) * 100. 385 c["size"] = (len(builderList[category]) * 100) / count 386 cs.append(c) 387 388 return cs
389
390 - def displaySlaveLine(self, status, builderList, debugInfo):
391 """Display a line the shows the current status for all the builders we 392 care about.""" 393 394 nbSlaves = 0 395 396 # Get the number of builders. 397 for category in builderList: 398 nbSlaves += len(builderList[category]) 399 400 # Get the categories, and order them alphabetically. 401 categories = builderList.keys() 402 categories.sort() 403 404 slaves = {} 405 406 # For each category, we display each builder. 407 for category in categories: 408 slaves[category] = [] 409 # For each builder in this category, we set the build info and we 410 # display the box. 411 for builder in builderList[category]: 412 s = {} 413 s["color"] = "notstarted" 414 s["title"] = builder 415 s["url"] = "./builders/%s" % urllib.quote(builder) 416 state, builds = status.getBuilder(builder).getState() 417 # Check if it's offline, if so, the box is purple. 418 if state == "offline": 419 s["color"] = "offline" 420 else: 421 # If not offline, then display the result of the last 422 # finished build. 423 build = self.getHeadBuild(status.getBuilder(builder)) 424 while build and not build.isFinished(): 425 build = build.getPreviousBuild() 426 427 if build: 428 s["color"] = getResultsClass(build.getResults(), None, 429 False) 430 431 slaves[category].append(s) 432 433 return slaves
434
435 - def displayStatusLine(self, builderList, allBuilds, revision, debugInfo):
436 """Display the boxes that represent the status of each builder in the 437 first build "revision" was in. Returns an HTML list of errors that 438 happened during these builds.""" 439 440 details = [] 441 nbSlaves = 0 442 for category in builderList: 443 nbSlaves += len(builderList[category]) 444 445 # Sort the categories. 446 categories = builderList.keys() 447 categories.sort() 448 449 builds = {} 450 451 # Display the boxes by category group. 452 for category in categories: 453 454 builds[category] = [] 455 456 # Display the boxes for each builder in this category. 457 for builder in builderList[category]: 458 introducedIn = None 459 firstNotIn = None 460 461 # Find the first build that does not include the revision. 462 for build in allBuilds[builder]: 463 if self.comparator.isRevisionEarlier(build, revision): 464 firstNotIn = build 465 break 466 else: 467 introducedIn = build 468 469 # Get the results of the first build with the revision, and the 470 # first build that does not include the revision. 471 results = None 472 previousResults = None 473 if introducedIn: 474 results = introducedIn.results 475 if firstNotIn: 476 previousResults = firstNotIn.results 477 478 isRunning = False 479 if introducedIn and not introducedIn.isFinished: 480 isRunning = True 481 482 url = "./waterfall" 483 title = builder 484 tag = "" 485 current_details = {} 486 if introducedIn: 487 current_details = introducedIn.details or "" 488 url = "./buildstatus?builder=%s&number=%s" % (urllib.quote(builder), 489 introducedIn.number) 490 title += " " 491 title += urllib.quote(' '.join(introducedIn.text), ' \n\\/:') 492 493 builderStrip = builder.replace(' ', '') 494 builderStrip = builderStrip.replace('(', '') 495 builderStrip = builderStrip.replace(')', '') 496 builderStrip = builderStrip.replace('.', '') 497 tag = "Tag%s%s" % (builderStrip, introducedIn.number) 498 499 if isRunning: 500 title += ' ETA: %ds' % (introducedIn.eta or 0) 501 502 resultsClass = getResultsClass(results, previousResults, isRunning) 503 504 b = {} 505 b["url"] = url 506 b["title"] = title 507 b["color"] = resultsClass 508 b["tag"] = tag 509 510 builds[category].append(b) 511 512 # If the box is red, we add the explaination in the details 513 # section. 514 if current_details and resultsClass == "failure": 515 details.append(current_details) 516 517 return (builds, details)
518
519 - def displayPage(self, request, status, builderList, allBuilds, revisions, 520 categories, branch, debugInfo):
521 """Display the console page.""" 522 # Build the main template directory with all the informations we have. 523 subs = dict() 524 subs["branch"] = branch or 'trunk' 525 if categories: 526 subs["categories"] = ' '.join(categories) 527 subs["time"] = time.strftime("%a %d %b %Y %H:%M:%S", 528 time.localtime(util.now())) 529 subs["debugInfo"] = debugInfo 530 subs["ANYBRANCH"] = ANYBRANCH 531 532 if builderList: 533 subs["categories"] = self.displayCategories(builderList, debugInfo) 534 subs['slaves'] = self.displaySlaveLine(status, builderList, debugInfo) 535 else: 536 subs["categories"] = [] 537 538 subs['revisions'] = [] 539 540 # For each revision we show one line 541 for revision in revisions: 542 r = {} 543 544 # Fill the dictionnary with these new information 545 r['id'] = revision.revision 546 r['link'] = revision.revlink 547 r['who'] = revision.who 548 r['date'] = revision.date 549 r['comments'] = revision.comments 550 r['repository'] = revision.repository 551 r['project'] = revision.project 552 553 # Display the status for all builders. 554 (builds, details) = self.displayStatusLine(builderList, 555 allBuilds, 556 revision, 557 debugInfo) 558 r['builds'] = builds 559 r['details'] = details 560 561 # Calculate the td span for the comment and the details. 562 r["span"] = len(builderList) + 2 563 564 subs['revisions'].append(r) 565 566 # 567 # Display the footer of the page. 568 # 569 debugInfo["load_time"] = time.time() - debugInfo["load_time"] 570 return subs
571 572
573 - def content(self, request, cxt):
574 "This method builds the main console view display." 575 576 reload_time = None 577 # Check if there was an arg. Don't let people reload faster than 578 # every 15 seconds. 0 means no reload. 579 if "reload" in request.args: 580 try: 581 reload_time = int(request.args["reload"][0]) 582 if reload_time != 0: 583 reload_time = max(reload_time, 15) 584 except ValueError: 585 pass 586 587 # Sets the default reload time to 60 seconds. 588 if not reload_time: 589 reload_time = 60 590 591 # Append the tag to refresh the page. 592 if reload_time is not None and reload_time != 0: 593 cxt['refresh'] = reload_time 594 595 # Debug information to display at the end of the page. 596 debugInfo = cxt['debuginfo'] = dict() 597 debugInfo["load_time"] = time.time() 598 599 # get url parameters 600 # Categories to show information for. 601 categories = request.args.get("category", []) 602 # List of all builders to show on the page. 603 builders = request.args.get("builder", []) 604 # Branch used to filter the changes shown. 605 branch = request.args.get("branch", [ANYBRANCH])[0] 606 # List of all the committers name to display on the page. 607 devName = request.args.get("name", []) 608 609 # and the data we want to render 610 status = self.getStatus(request) 611 612 # Get all revisions we can find. 613 source = self.getChangeManager(request) 614 allChanges = self.getAllChanges(source, status, debugInfo) 615 616 debugInfo["source_all"] = len(allChanges) 617 618 # Keep only the revisions we care about. 619 # By default we process the last 40 revisions. 620 # If a dev name is passed, we look for the changes by this person in the 621 # last 80 revisions. 622 numRevs = 40 623 if devName: 624 numRevs *= 2 625 numBuilds = numRevs 626 627 628 revisions = self.stripRevisions(allChanges, numRevs, branch, devName) 629 debugInfo["revision_final"] = len(revisions) 630 631 # Fetch all the builds for all builders until we get the next build 632 # after lastRevision. 633 builderList = None 634 allBuilds = None 635 if revisions: 636 lastRevision = revisions[len(revisions) - 1].revision 637 debugInfo["last_revision"] = lastRevision 638 639 (builderList, allBuilds) = self.getAllBuildsForRevision(status, 640 request, 641 lastRevision, 642 numBuilds, 643 categories, 644 builders, 645 debugInfo) 646 647 debugInfo["added_blocks"] = 0 648 649 cxt.update(self.displayPage(request, status, builderList, allBuilds, 650 revisions, categories, branch, debugInfo)) 651 652 template = request.site.buildbot_service.templates.get_template("console.html") 653 data = template.render(cxt) 654 return data
655
656 -class RevisionComparator(object):
657 """Used for comparing between revisions, as some 658 VCS use a plain counter for revisions (like SVN) 659 while others use different concepts (see Git). 660 """ 661 662 # TODO (avivby): Should this be a zope interface? 663
664 - def isRevisionEarlier(self, first_change, second_change):
665 """Used for comparing 2 changes""" 666 raise NotImplementedError
667
668 - def isValidRevision(self, revision):
669 """Checks whether the revision seems like a VCS revision""" 670 raise NotImplementedError
671
672 - def getSortingKey(self):
673 raise NotImplementedError
674
675 -class TimeRevisionComparator(RevisionComparator):
676 - def isRevisionEarlier(self, first, second):
677 return first.when < second.when
678
679 - def isValidRevision(self, revision):
680 return True # No general way of determining
681
682 - def getSortingKey(self):
683 return operator.attrgetter('when')
684
685 -class IntegerRevisionComparator(RevisionComparator):
686 - def isRevisionEarlier(self, first, second):
687 return int(first.revision) < int(second.revision)
688
689 - def isValidRevision(self, revision):
690 try: 691 int(revision) 692 return True 693 except: 694 return False
695
696 - def getSortingKey(self):
697 return operator.attrgetter('revision')
698