Source Code for Module buildbot.status.web.console

  1  from __future__ import generators 
  2   
  3  import time 
  4  import operator 
  5  import re 
  6  import urllib 
  7   
  8  from buildbot import util 
  9  from buildbot.status import builder 
 10  from buildbot.status.web.base import HtmlResource 
 11   
 12 -def getResultsClass(results, prevResults, inProgress): 
 13      """Given the current and past results, return the class that will be used 
 14      by the css to display the right color for a box.""" 
 15   
 16      if inProgress: 
 17          return "running" 
 18   
 19      if results is None: 
 20          return "notstarted" 
 21   
 22      if results == builder.SUCCESS: 
 23          return "success" 
 24   
 25      if results == builder.FAILURE: 
 26          if not prevResults: 
 27              # This is the bottom box. We don't know if the previous one failed 
 28              # or not. We assume it did not. 
 29              return "failure" 
 30   
 31          if prevResults != builder.FAILURE: 
 32              # This is a new failure. 
 33              return "failure" 
 34          else: 
 35              # The previous build also failed. 
 36              return "warnings" 
 37     
 38      # Any other results? Like EXCEPTION? 
 39      return "exception" 
 40   
 41 -class ANYBRANCH: pass # a flag value, used below 
 42   
 43 -class DevRevision: 
 44      """Helper class that contains all the information we need for a revision.""" 
 45   
 46 -    def __init__(self, change): 
 47          self.revision = change.revision 
 48          self.comments = change.comments 
 49          self.who = change.who 
 50          self.date = change.getTime() 
 51          self.revlink = getattr(change, 'revlink', None) 
 52          self.when = change.when 
 53          self.repository = change.repository 
 54          self.project = change.project 
 55   
 56   
 57 -class DevBuild: 
 58      """Helper class that contains all the information we need for a build.""" 
 59   
 60 -    def __init__(self, revision, build, details): 
 61          self.revision = revision 
 62          self.results =  build.getResults() 
 63          self.number = build.getNumber() 
 64          self.isFinished = build.isFinished() 
 65          self.text = build.getText() 
 66          self.eta = build.getETA() 
 67          self.details = details 
 68          self.when = build.getTimes()[0] 
 69          self.source = build.getSourceStamp() 
 70   
 71   
 72 -class ConsoleStatusResource(HtmlResource): 
 73      """Main console class. It displays a user-oriented status page. 
 74      Every change is a line in the page, and it shows the result of the first 
 75      build with this change for each slave.""" 
 76   
 77 -    def __init__(self, orderByTime=False): 
 78          HtmlResource.__init__(self) 
 79   
 80          self.status = None 
 81   
 82          if orderByTime: 
 83              self.comparator = TimeRevisionComparator() 
 84          else: 
 85              self.comparator = IntegerRevisionComparator() 
 86   
 87 -    def getTitle(self, request): 
 88          status = self.getStatus(request) 
 89          projectName = status.getProjectName() 
 90          if projectName: 
 91              return "BuildBot: %s" % projectName 
 92          else: 
 93              return "BuildBot" 
 94   
 95 -    def getChangeManager(self, request): 
 96          return request.site.buildbot_service.parent.change_svc 
 97   
 98      ## 
 99      ## Data gathering functions 
100      ## 
101   
102 -    def getHeadBuild(self, builder): 
103          """Get the most recent build for the given builder. 
104          """ 
105          build = builder.getBuild(-1) 
106   
107          # HACK: Work around #601, the head build may be None if it is 
108          # locked. 
109          if build is None: 
110              build = builder.getBuild(-2) 
111   
112          return build 
113   
114 -    def fetchChangesFromHistory(self, status, max_depth, max_builds, debugInfo): 
115          """Look at the history of the builders and try to fetch as many changes 
116          as possible. We need this when the main source does not contain enough 
117          sourcestamps.  
118   
119          max_depth defines how many builds we will parse for a given builder. 
120          max_builds defines how many builds total we want to parse. This is to 
121              limit the amount of time we spend in this function. 
122           
123          This function is sub-optimal, but the information returned by this 
124          function is cached, so this function won't be called more than once. 
125          """ 
126           
127          allChanges = list() 
128          build_count = 0 
129          for builderName in status.getBuilderNames()[:]: 
130              if build_count > max_builds: 
131                  break 
132               
133              builder = status.getBuilder(builderName) 
134              build = self.getHeadBuild(builder) 
135              depth = 0 
136              while build and depth < max_depth and build_count < max_builds: 
137                  depth += 1 
138                  build_count += 1 
139                  sourcestamp = build.getSourceStamp() 
140                  allChanges.extend(sourcestamp.changes[:]) 
141                  build = build.getPreviousBuild() 
142   
143          debugInfo["source_fetch_len"] = len(allChanges) 
144          return allChanges                 
145   
146 -    def getAllChanges(self, source, status, debugInfo): 
147          """Return all the changes we can find at this time. If |source| does not 
148          not have enough (less than 25), we try to fetch more from the builders 
149          history.""" 
150   
151          g = source.eventGenerator() 
152          allChanges = [] 
153          while len(allChanges) < 25: 
154              try: 
155                  c = g.next() 
156              except StopIteration: 
157                  break 
158              allChanges.append(c) 
159   
160          allChanges.sort(key=self.comparator.getSortingKey()) 
161   
162          # Remove the dups 
163          prevChange = None 
164          newChanges = [] 
165          for change in allChanges: 
166              rev = change.revision 
167              if not prevChange or rev != prevChange.revision: 
168                  newChanges.append(change) 
169              prevChange = change 
170          allChanges = newChanges 
171   
172          return allChanges 
173   
174 -    def stripRevisions(self, allChanges, numRevs, branch, devName): 
175          """Returns a subset of changes from allChanges that matches the query. 
176   
177          allChanges is the list of all changes we know about. 
178          numRevs is the number of changes we will inspect from allChanges. We 
179              do not want to inspect all of them or it would be too slow. 
180          branch is the branch we are interested in. Changes not in this branch 
181              will be ignored. 
182          devName is the developper name. Changes have not been submitted by this 
183              person will be ignored. 
184          """ 
185           
186          revisions = [] 
187   
188          if not allChanges: 
189              return revisions 
190   
191          totalRevs = len(allChanges) 
192          for i in range(totalRevs - 1, totalRevs - numRevs, -1): 
193              if i < 0: 
194                  break 
195              change = allChanges[i] 
196              if branch == ANYBRANCH or branch == change.branch: 
197                  if not devName or change.who in devName:                     
198                      rev = DevRevision(change) 
199                      revisions.append(rev) 
200   
201          return revisions 
202   
203 -    def getBuildDetails(self, request, builderName, build): 
204          """Returns an HTML list of failures for a given build.""" 
205          details = {} 
206          if not build.getLogs(): 
207              return details 
208           
209          for step in build.getSteps(): 
210              (result, reason) = step.getResults() 
211              if result == builder.FAILURE: 
212                  name = step.getName() 
213   
214                  # Remove html tags from the error text. 
215                  stripHtml = re.compile(r'<.*?>') 
216                  strippedDetails = stripHtml.sub('', ' '.join(step.getText())) 
217                   
218                  details['buildername'] = builderName 
219                  details['status'] = strippedDetails 
220                  details['reason'] = reason 
221                  logs = details['logs'] = [] 
222   
223                  if step.getLogs(): 
224                      for log in step.getLogs(): 
225                          logname = log.getName() 
226                          logurl = request.childLink( 
227                            "../builders/%s/builds/%s/steps/%s/logs/%s" %  
228                              (urllib.quote(builderName), 
229                               build.getNumber(), 
230                               urllib.quote(name), 
231                               urllib.quote(logname))) 
232                          logs.append(dict(url=logurl, name=logname)) 
233          return details 
234   
235 -    def getBuildsForRevision(self, request, builder, builderName, lastRevision, 
236                               numBuilds, debugInfo): 
237          """Return the list of all the builds for a given builder that we will 
238          need to be able to display the console page. We start by the most recent 
239          build, and we go down until we find a build that was built prior to the 
240          last change we are interested in.""" 
241   
242          revision = lastRevision  
243   
244          builds = [] 
245          build = self.getHeadBuild(builder) 
246          number = 0 
247          while build and number < numBuilds: 
248              debugInfo["builds_scanned"] += 1 
249              number += 1 
250   
251              # Get the last revision in this build. 
252              # We first try "got_revision", but if it does not work, then 
253              # we try "revision". 
254              got_rev = -1 
255              try: 
256                  got_rev = build.getProperty("got_revision") 
257                  if not self.comparator.isValidRevision(got_rev): 
258                      got_rev = -1 
259              except KeyError: 
260                  pass 
261   
262              try: 
263                  if got_rev == -1: 
264                      got_rev = build.getProperty("revision") 
265                  if not self.comparator.isValidRevision(got_rev): 
266                      got_rev = -1 
267              except: 
268                  pass 
269   
270              # We ignore all builds that don't have last revisions. 
271              # TODO(nsylvain): If the build is over, maybe it was a problem 
272              # with the update source step. We need to find a way to tell the 
273              # user that his change might have broken the source update. 
274              if got_rev and got_rev != -1: 
275                  details = self.getBuildDetails(request, builderName, build) 
276                  devBuild = DevBuild(got_rev, build, details) 
277                  builds.append(devBuild) 
278   
279                  # Now break if we have enough builds. 
280                  current_revision = self.getChangeForBuild( 
281                      build, revision) 
282                  if self.comparator.isRevisionEarlier( 
283                      devBuild, current_revision): 
284                      break 
285   
286              build = build.getPreviousBuild() 
287   
288          return builds 
289   
290 -    def getChangeForBuild(self, build, revision): 
291          if not build or not build.getChanges(): # Forced build 
292              return DevBuild(revision, build, None) 
293           
294          for change in build.getChanges(): 
295              if change.revision == revision: 
296                  return change 
297   
298          # No matching change, return the last change in build. 
299          changes = list(build.getChanges()) 
300          changes.sort(key=self.comparator.getSortingKey()) 
301          return changes[-1] 
302       
303 -    def getAllBuildsForRevision(self, status, request, lastRevision, numBuilds, 
304                                  categories, builders, debugInfo): 
305          """Returns a dictionnary of builds we need to inspect to be able to 
306          display the console page. The key is the builder name, and the value is 
307          an array of build we care about. We also returns a dictionnary of 
308          builders we care about. The key is it's category. 
309    
310          lastRevision is the last revision we want to display in the page. 
311          categories is a list of categories to display. It is coming from the 
312              HTTP GET parameters. 
313          builders is a list of builders to display. It is coming from the HTTP 
314              GET parameters. 
315          """ 
316   
317          allBuilds = dict() 
318   
319          # List of all builders in the dictionnary. 
320          builderList = dict() 
321   
322          debugInfo["builds_scanned"] = 0 
323          # Get all the builders. 
324          builderNames = status.getBuilderNames()[:] 
325          for builderName in builderNames: 
326              builder = status.getBuilder(builderName) 
327   
328              # Make sure we are interested in this builder. 
329              if categories and builder.category not in categories: 
330                  continue 
331              if builders and builderName not in builders: 
332                  continue 
333   
334              # We want to display this builder. 
335              category = builder.category or "default" 
336              # Strip the category to keep only the text before the first |. 
337              # This is a hack to support the chromium usecase where they have 
338              # multiple categories for each slave. We use only the first one. 
339              # TODO(nsylvain): Create another way to specify "display category" 
340              #     in master.cfg. 
341              category = category.split('|')[0] 
342              if not builderList.get(category): 
343                  builderList[category] = [] 
344   
345              # Append this builder to the dictionnary of builders. 
346              builderList[category].append(builderName) 
347              # Set the list of builds for this builder. 
348              allBuilds[builderName] = self.getBuildsForRevision(request, 
349                                                                 builder, 
350                                                                 builderName, 
351                                                                 lastRevision, 
352                                                                 numBuilds, 
353                                                                 debugInfo) 
354   
355          return (builderList, allBuilds) 
356   
357   
358      ## 
359      ## Display functions 
360      ## 
361   
362 -    def displayCategories(self, builderList, debugInfo): 
363          """Display the top category line.""" 
364   
365          count = 0 
366          for category in builderList: 
367              count += len(builderList[category]) 
368   
369          categories = builderList.keys() 
370          categories.sort() 
371           
372          cs = [] 
373           
374          for category in categories:             
375              c = {} 
376              # TODO(nsylvain): Another hack to display the category in a pretty 
377              # way.  If the master owner wants to display the categories in a 
378              # given order, he/she can prepend a number to it. This number won't 
379              # be shown. 
380              c["name"] = category.lstrip('0123456789') 
381   
382              # To be able to align the table correctly, we need to know 
383              # what percentage of space this category will be taking. This is 
384              # (#Builders in Category) / (#Builders Total) * 100. 
385              c["size"] = (len(builderList[category]) * 100) / count             
386              cs.append(c) 
387               
388          return cs 
389   
390 -    def displaySlaveLine(self, status, builderList, debugInfo): 
391          """Display a line the shows the current status for all the builders we 
392          care about.""" 
393   
394          nbSlaves = 0 
395   
396          # Get the number of builders. 
397          for category in builderList: 
398              nbSlaves += len(builderList[category]) 
399   
400          # Get the categories, and order them alphabetically. 
401          categories = builderList.keys() 
402          categories.sort() 
403   
404          slaves = {} 
405   
406          # For each category, we display each builder. 
407          for category in categories: 
408              slaves[category] = [] 
409              # For each builder in this category, we set the build info and we 
410              # display the box. 
411              for builder in builderList[category]: 
412                  s = {} 
413                  s["color"] = "notstarted" 
414                  s["title"] = builder 
415                  s["url"] = "./builders/%s" % urllib.quote(builder) 
416                  state, builds = status.getBuilder(builder).getState() 
417                  # Check if it's offline, if so, the box is purple. 
418                  if state == "offline": 
419                      s["color"] = "offline" 
420                  else: 
421                      # If not offline, then display the result of the last 
422                      # finished build. 
423                      build = self.getHeadBuild(status.getBuilder(builder)) 
424                      while build and not build.isFinished(): 
425                          build = build.getPreviousBuild() 
426   
427                      if build: 
428                          s["color"] = getResultsClass(build.getResults(), None, 
429                                                        False) 
430   
431                  slaves[category].append(s) 
432   
433          return slaves 
434   
435 -    def displayStatusLine(self, builderList, allBuilds, revision, debugInfo): 
436          """Display the boxes that represent the status of each builder in the 
437          first build "revision" was in. Returns an HTML list of errors that 
438          happened during these builds.""" 
439   
440          details = [] 
441          nbSlaves = 0 
442          for category in builderList: 
443              nbSlaves += len(builderList[category]) 
444   
445          # Sort the categories. 
446          categories = builderList.keys() 
447          categories.sort() 
448           
449          builds = {} 
450     
451          # Display the boxes by category group. 
452          for category in categories: 
453     
454              builds[category] = [] 
455               
456              # Display the boxes for each builder in this category. 
457              for builder in builderList[category]: 
458                  introducedIn = None 
459                  firstNotIn = None 
460   
461                  # Find the first build that does not include the revision. 
462                  for build in allBuilds[builder]: 
463                      if self.comparator.isRevisionEarlier(build, revision): 
464                          firstNotIn = build 
465                          break 
466                      else: 
467                          introducedIn = build 
468                           
469                  # Get the results of the first build with the revision, and the 
470                  # first build that does not include the revision. 
471                  results = None 
472                  previousResults = None 
473                  if introducedIn: 
474                      results = introducedIn.results 
475                  if firstNotIn: 
476                      previousResults = firstNotIn.results 
477   
478                  isRunning = False 
479                  if introducedIn and not introducedIn.isFinished: 
480                      isRunning = True 
481   
482                  url = "./waterfall" 
483                  title = builder 
484                  tag = "" 
485                  current_details = {} 
486                  if introducedIn: 
487                      current_details = introducedIn.details or "" 
488                      url = "./buildstatus?builder=%s&number=%s" % (urllib.quote(builder), 
489                                                                    introducedIn.number) 
490                      title += " " 
491                      title += urllib.quote(' '.join(introducedIn.text), ' \n\\/:') 
492   
493                      builderStrip = builder.replace(' ', '') 
494                      builderStrip = builderStrip.replace('(', '') 
495                      builderStrip = builderStrip.replace(')', '') 
496                      builderStrip = builderStrip.replace('.', '') 
497                      tag = "Tag%s%s" % (builderStrip, introducedIn.number) 
498   
499                  if isRunning: 
500                      title += ' ETA: %ds' % (introducedIn.eta or 0) 
501                       
502                  resultsClass = getResultsClass(results, previousResults, isRunning) 
503   
504                  b = {}                 
505                  b["url"] = url 
506                  b["title"] = title 
507                  b["color"] = resultsClass 
508                  b["tag"] = tag 
509   
510                  builds[category].append(b) 
511   
512                  # If the box is red, we add the explaination in the details 
513                  # section. 
514                  if current_details and resultsClass == "failure": 
515                      details.append(current_details) 
516   
517          return (builds, details) 
518   
519 -    def displayPage(self, request, status, builderList, allBuilds, revisions, 
520                      categories, branch, debugInfo): 
521          """Display the console page.""" 
522          # Build the main template directory with all the informations we have. 
523          subs = dict() 
524          subs["branch"] = branch or 'trunk' 
525          if categories: 
526              subs["categories"] = ' '.join(categories) 
527          subs["time"] = time.strftime("%a %d %b %Y %H:%M:%S", 
528                                       time.localtime(util.now())) 
529          subs["debugInfo"] = debugInfo 
530          subs["ANYBRANCH"] = ANYBRANCH 
531   
532          if builderList: 
533              subs["categories"] = self.displayCategories(builderList, debugInfo) 
534              subs['slaves'] = self.displaySlaveLine(status, builderList, debugInfo) 
535          else: 
536              subs["categories"] = [] 
537   
538          subs['revisions'] = [] 
539   
540          # For each revision we show one line 
541          for revision in revisions: 
542              r = {} 
543               
544              # Fill the dictionnary with these new information 
545              r['id'] = revision.revision 
546              r['link'] = revision.revlink  
547              r['who'] = revision.who 
548              r['date'] = revision.date 
549              r['comments'] = revision.comments 
550              r['repository'] = revision.repository 
551              r['project'] = revision.project 
552   
553              # Display the status for all builders. 
554              (builds, details) = self.displayStatusLine(builderList, 
555                                              allBuilds, 
556                                              revision, 
557                                              debugInfo) 
558              r['builds'] = builds 
559              r['details'] = details 
560   
561              # Calculate the td span for the comment and the details. 
562              r["span"] = len(builderList) + 2             
563   
564              subs['revisions'].append(r) 
565   
566          # 
567          # Display the footer of the page. 
568          # 
569          debugInfo["load_time"] = time.time() - debugInfo["load_time"] 
570          return subs 
571   
572   
573 -    def content(self, request, cxt): 
574          "This method builds the main console view display." 
575   
576          reload_time = None 
577          # Check if there was an arg. Don't let people reload faster than 
578          # every 15 seconds. 0 means no reload. 
579          if "reload" in request.args: 
580              try: 
581                  reload_time = int(request.args["reload"][0]) 
582                  if reload_time != 0: 
583                      reload_time = max(reload_time, 15) 
584              except ValueError: 
585                  pass 
586   
587          # Sets the default reload time to 60 seconds. 
588          if not reload_time: 
589              reload_time = 60 
590   
591          # Append the tag to refresh the page.  
592          if reload_time is not None and reload_time != 0: 
593              cxt['refresh'] = reload_time 
594   
595          # Debug information to display at the end of the page. 
596          debugInfo = cxt['debuginfo'] = dict() 
597          debugInfo["load_time"] = time.time() 
598   
599          # get url parameters 
600          # Categories to show information for. 
601          categories = request.args.get("category", []) 
602          # List of all builders to show on the page. 
603          builders = request.args.get("builder", []) 
604          # Branch used to filter the changes shown. 
605          branch = request.args.get("branch", [ANYBRANCH])[0] 
606          # List of all the committers name to display on the page. 
607          devName = request.args.get("name", []) 
608   
609          # and the data we want to render 
610          status = self.getStatus(request) 
611   
612          # Get all revisions we can find. 
613          source = self.getChangeManager(request) 
614          allChanges = self.getAllChanges(source, status, debugInfo) 
615   
616          debugInfo["source_all"] = len(allChanges) 
617   
618          # Keep only the revisions we care about. 
619          # By default we process the last 40 revisions. 
620          # If a dev name is passed, we look for the changes by this person in the 
621          # last 80 revisions. 
622          numRevs = 40 
623          if devName: 
624              numRevs *= 2 
625          numBuilds = numRevs 
626   
627   
628          revisions = self.stripRevisions(allChanges, numRevs, branch, devName) 
629          debugInfo["revision_final"] = len(revisions) 
630   
631          # Fetch all the builds for all builders until we get the next build 
632          # after lastRevision. 
633          builderList = None 
634          allBuilds = None 
635          if revisions: 
636              lastRevision = revisions[len(revisions) - 1].revision 
637              debugInfo["last_revision"] = lastRevision 
638   
639              (builderList, allBuilds) = self.getAllBuildsForRevision(status, 
640                                                  request, 
641                                                  lastRevision, 
642                                                  numBuilds, 
643                                                  categories, 
644                                                  builders, 
645                                                  debugInfo) 
646   
647          debugInfo["added_blocks"] = 0 
648   
649          cxt.update(self.displayPage(request, status, builderList, allBuilds, 
650                                      revisions, categories, branch, debugInfo)) 
651   
652          template = request.site.buildbot_service.templates.get_template("console.html") 
653          data = template.render(cxt) 
654          return data 
655   
656 -class RevisionComparator(object): 
657      """Used for comparing between revisions, as some 
658      VCS use a plain counter for revisions (like SVN) 
659      while others use different concepts (see Git). 
660      """ 
661       
662      # TODO (avivby): Should this be a zope interface? 
663       
664 -    def isRevisionEarlier(self, first_change, second_change): 
665          """Used for comparing 2 changes""" 
666          raise NotImplementedError 
667   
668 -    def isValidRevision(self, revision): 
669          """Checks whether the revision seems like a VCS revision""" 
670          raise NotImplementedError 
671   
672 -    def getSortingKey(self): 
673          raise NotImplementedError 
674       
675 -class TimeRevisionComparator(RevisionComparator): 
676 -    def isRevisionEarlier(self, first, second): 
677          return first.when < second.when 
678   
679 -    def isValidRevision(self, revision): 
680          return True # No general way of determining 
681   
682 -    def getSortingKey(self): 
683          return operator.attrgetter('when') 
684   
685 -class IntegerRevisionComparator(RevisionComparator): 
686 -    def isRevisionEarlier(self, first, second): 
687          return int(first.revision) < int(second.revision) 
688   
689 -    def isValidRevision(self, revision): 
690          try: 
691              int(revision) 
692              return True 
693          except: 
694              return False 
695   
696 -    def getSortingKey(self): 
697          return operator.attrgetter('revision') 
698