Source Code for Module buildbot.status.web.console

  1  # This file is part of Buildbot.  Buildbot is free software: you can 
  2  # redistribute it and/or modify it under the terms of the GNU General Public 
  3  # License as published by the Free Software Foundation, version 2. 
  4  # 
  5  # This program is distributed in the hope that it will be useful, but WITHOUT 
  6  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
  7  # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more 
  8  # details. 
  9  # 
 10  # You should have received a copy of the GNU General Public License along with 
 11  # this program; if not, write to the Free Software Foundation, Inc., 51 
 12  # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 
 13  # 
 14  # Copyright Buildbot Team Members 
 15   
 16  from __future__ import generators 
 17   
 18  import time 
 19  import operator 
 20  import re 
 21  import urllib 
 22   
 23  from buildbot import util 
 24  from buildbot.status import builder 
 25  from buildbot.status.web.base import HtmlResource 
 26   
 27 -class DoesNotPassFilter(Exception): pass # Used for filtering revs 
 28   
 29 -def getResultsClass(results, prevResults, inProgress): 
 30      """Given the current and past results, return the class that will be used 
 31      by the css to display the right color for a box.""" 
 32   
 33      if inProgress: 
 34          return "running" 
 35   
 36      if results is None: 
 37          return "notstarted" 
 38   
 39      if results == builder.SUCCESS: 
 40          return "success" 
 41   
 42      if results == builder.WARNINGS: 
 43          return "warnings" 
 44   
 45      if results == builder.FAILURE: 
 46          if not prevResults: 
 47              # This is the bottom box. We don't know if the previous one failed 
 48              # or not. We assume it did not. 
 49              return "failure" 
 50   
 51          if prevResults != builder.FAILURE: 
 52              # This is a new failure. 
 53              return "failure" 
 54          else: 
 55              # The previous build also failed. 
 56              return "warnings" 
 57     
 58      # Any other results? Like EXCEPTION? 
 59      return "exception" 
 60   
 61 -class ANYBRANCH: pass # a flag value, used below 
 62   
 63 -class DevRevision: 
 64      """Helper class that contains all the information we need for a revision.""" 
 65   
 66 -    def __init__(self, change): 
 67          self.revision = change.revision 
 68          self.comments = change.comments 
 69          self.who = change.who 
 70          self.date = change.getTime() 
 71          self.revlink = getattr(change, 'revlink', None) 
 72          self.when = change.when 
 73          self.repository = change.repository 
 74          self.project = change.project 
 75   
 76   
 77 -class DevBuild: 
 78      """Helper class that contains all the information we need for a build.""" 
 79   
 80 -    def __init__(self, revision, build, details): 
 81          self.revision = revision 
 82          self.results =  build.getResults() 
 83          self.number = build.getNumber() 
 84          self.isFinished = build.isFinished() 
 85          self.text = build.getText() 
 86          self.eta = build.getETA() 
 87          self.details = details 
 88          self.when = build.getTimes()[0] 
 89          self.source = build.getSourceStamp() 
 90   
 91   
 92 -class ConsoleStatusResource(HtmlResource): 
 93      """Main console class. It displays a user-oriented status page. 
 94      Every change is a line in the page, and it shows the result of the first 
 95      build with this change for each slave.""" 
 96   
 97 -    def __init__(self, orderByTime=False): 
 98          HtmlResource.__init__(self) 
 99   
100          self.status = None 
101   
102          if orderByTime: 
103              self.comparator = TimeRevisionComparator() 
104          else: 
105              self.comparator = IntegerRevisionComparator() 
106   
107 -    def getTitle(self, request): 
108          status = self.getStatus(request) 
109          projectName = status.getProjectName() 
110          if projectName: 
111              return "BuildBot: %s" % projectName 
112          else: 
113              return "BuildBot" 
114   
115 -    def getChangeManager(self, request): 
116          return request.site.buildbot_service.parent.change_svc 
117   
118      ## 
119      ## Data gathering functions 
120      ## 
121   
122 -    def getHeadBuild(self, builder): 
123          """Get the most recent build for the given builder. 
124          """ 
125          build = builder.getBuild(-1) 
126   
127          # HACK: Work around #601, the head build may be None if it is 
128          # locked. 
129          if build is None: 
130              build = builder.getBuild(-2) 
131   
132          return build 
133   
134 -    def fetchChangesFromHistory(self, status, max_depth, max_builds, debugInfo): 
135          """Look at the history of the builders and try to fetch as many changes 
136          as possible. We need this when the main source does not contain enough 
137          sourcestamps.  
138   
139          max_depth defines how many builds we will parse for a given builder. 
140          max_builds defines how many builds total we want to parse. This is to 
141              limit the amount of time we spend in this function. 
142           
143          This function is sub-optimal, but the information returned by this 
144          function is cached, so this function won't be called more than once. 
145          """ 
146           
147          allChanges = list() 
148          build_count = 0 
149          for builderName in status.getBuilderNames()[:]: 
150              if build_count > max_builds: 
151                  break 
152               
153              builder = status.getBuilder(builderName) 
154              build = self.getHeadBuild(builder) 
155              depth = 0 
156              while build and depth < max_depth and build_count < max_builds: 
157                  depth += 1 
158                  build_count += 1 
159                  sourcestamp = build.getSourceStamp() 
160                  allChanges.extend(sourcestamp.changes[:]) 
161                  build = build.getPreviousBuild() 
162   
163          debugInfo["source_fetch_len"] = len(allChanges) 
164          return allChanges                 
165   
166 -    def getAllChanges(self, source, status, debugInfo): 
167          """Return all the changes we can find at this time. If |source| does not 
168          not have enough (less than 25), we try to fetch more from the builders 
169          history.""" 
170   
171          g = source.eventGenerator() 
172          allChanges = [] 
173          while len(allChanges) < 25: 
174              try: 
175                  c = g.next() 
176              except StopIteration: 
177                  break 
178              allChanges.append(c) 
179   
180          allChanges.sort(key=self.comparator.getSortingKey()) 
181   
182          # Remove the dups 
183          prevChange = None 
184          newChanges = [] 
185          for change in allChanges: 
186              rev = change.revision 
187              if not prevChange or rev != prevChange.revision: 
188                  newChanges.append(change) 
189              prevChange = change 
190          allChanges = newChanges 
191   
192          return allChanges 
193   
194 -    def getBuildDetails(self, request, builderName, build): 
195          """Returns an HTML list of failures for a given build.""" 
196          details = {} 
197          if not build.getLogs(): 
198              return details 
199           
200          for step in build.getSteps(): 
201              (result, reason) = step.getResults() 
202              if result == builder.FAILURE: 
203                  name = step.getName() 
204   
205                  # Remove html tags from the error text. 
206                  stripHtml = re.compile(r'<.*?>') 
207                  strippedDetails = stripHtml.sub('', ' '.join(step.getText())) 
208                   
209                  details['buildername'] = builderName 
210                  details['status'] = strippedDetails 
211                  details['reason'] = reason 
212                  logs = details['logs'] = [] 
213   
214                  if step.getLogs(): 
215                      for log in step.getLogs(): 
216                          logname = log.getName() 
217                          logurl = request.childLink( 
218                            "../builders/%s/builds/%s/steps/%s/logs/%s" %  
219                              (urllib.quote(builderName), 
220                               build.getNumber(), 
221                               urllib.quote(name), 
222                               urllib.quote(logname))) 
223                          logs.append(dict(url=logurl, name=logname)) 
224          return details 
225   
226 -    def getBuildsForRevision(self, request, builder, builderName, lastRevision, 
227                               numBuilds, debugInfo): 
228          """Return the list of all the builds for a given builder that we will 
229          need to be able to display the console page. We start by the most recent 
230          build, and we go down until we find a build that was built prior to the 
231          last change we are interested in.""" 
232   
233          revision = lastRevision  
234   
235          builds = [] 
236          build = self.getHeadBuild(builder) 
237          number = 0 
238          while build and number < numBuilds: 
239              debugInfo["builds_scanned"] += 1 
240              number += 1 
241   
242              # Get the last revision in this build. 
243              # We first try "got_revision", but if it does not work, then 
244              # we try "revision". 
245              got_rev = -1 
246              try: 
247                  got_rev = build.getProperty("got_revision") 
248                  if not self.comparator.isValidRevision(got_rev): 
249                      got_rev = -1 
250              except KeyError: 
251                  pass 
252   
253              try: 
254                  if got_rev == -1: 
255                      got_rev = build.getProperty("revision") 
256                  if not self.comparator.isValidRevision(got_rev): 
257                      got_rev = -1 
258              except: 
259                  pass 
260   
261              # We ignore all builds that don't have last revisions. 
262              # TODO(nsylvain): If the build is over, maybe it was a problem 
263              # with the update source step. We need to find a way to tell the 
264              # user that his change might have broken the source update. 
265              if got_rev and got_rev != -1: 
266                  details = self.getBuildDetails(request, builderName, build) 
267                  devBuild = DevBuild(got_rev, build, details) 
268                  builds.append(devBuild) 
269   
270                  # Now break if we have enough builds. 
271                  current_revision = self.getChangeForBuild( 
272                      build, revision) 
273                  if self.comparator.isRevisionEarlier( 
274                      devBuild, current_revision): 
275                      break 
276   
277              build = build.getPreviousBuild() 
278   
279          return builds 
280   
281 -    def getChangeForBuild(self, build, revision): 
282          if not build or not build.getChanges(): # Forced build 
283              return DevBuild(revision, build, None) 
284           
285          for change in build.getChanges(): 
286              if change.revision == revision: 
287                  return change 
288   
289          # No matching change, return the last change in build. 
290          changes = list(build.getChanges()) 
291          changes.sort(key=self.comparator.getSortingKey()) 
292          return changes[-1] 
293       
294 -    def getAllBuildsForRevision(self, status, request, lastRevision, numBuilds, 
295                                  categories, builders, debugInfo): 
296          """Returns a dictionary of builds we need to inspect to be able to 
297          display the console page. The key is the builder name, and the value is 
298          an array of build we care about. We also returns a dictionary of 
299          builders we care about. The key is it's category. 
300    
301          lastRevision is the last revision we want to display in the page. 
302          categories is a list of categories to display. It is coming from the 
303              HTTP GET parameters. 
304          builders is a list of builders to display. It is coming from the HTTP 
305              GET parameters. 
306          """ 
307   
308          allBuilds = dict() 
309   
310          # List of all builders in the dictionary. 
311          builderList = dict() 
312   
313          debugInfo["builds_scanned"] = 0 
314          # Get all the builders. 
315          builderNames = status.getBuilderNames()[:] 
316          for builderName in builderNames: 
317              builder = status.getBuilder(builderName) 
318   
319              # Make sure we are interested in this builder. 
320              if categories and builder.category not in categories: 
321                  continue 
322              if builders and builderName not in builders: 
323                  continue 
324   
325              # We want to display this builder. 
326              category = builder.category or "default" 
327              # Strip the category to keep only the text before the first |. 
328              # This is a hack to support the chromium usecase where they have 
329              # multiple categories for each slave. We use only the first one. 
330              # TODO(nsylvain): Create another way to specify "display category" 
331              #     in master.cfg. 
332              category = category.split('|')[0] 
333              if not builderList.get(category): 
334                  builderList[category] = [] 
335   
336              # Append this builder to the dictionary of builders. 
337              builderList[category].append(builderName) 
338              # Set the list of builds for this builder. 
339              allBuilds[builderName] = self.getBuildsForRevision(request, 
340                                                                 builder, 
341                                                                 builderName, 
342                                                                 lastRevision, 
343                                                                 numBuilds, 
344                                                                 debugInfo) 
345   
346          return (builderList, allBuilds) 
347   
348   
349      ## 
350      ## Display functions 
351      ## 
352   
353 -    def displayCategories(self, builderList, debugInfo): 
354          """Display the top category line.""" 
355   
356          count = 0 
357          for category in builderList: 
358              count += len(builderList[category]) 
359   
360          categories = builderList.keys() 
361          categories.sort() 
362           
363          cs = [] 
364           
365          for category in categories:             
366              c = {} 
367   
368              c["name"] = category 
369   
370              # To be able to align the table correctly, we need to know 
371              # what percentage of space this category will be taking. This is 
372              # (#Builders in Category) / (#Builders Total) * 100. 
373              c["size"] = (len(builderList[category]) * 100) / count             
374              cs.append(c) 
375               
376          return cs 
377   
378 -    def displaySlaveLine(self, status, builderList, debugInfo): 
379          """Display a line the shows the current status for all the builders we 
380          care about.""" 
381   
382          nbSlaves = 0 
383   
384          # Get the number of builders. 
385          for category in builderList: 
386              nbSlaves += len(builderList[category]) 
387   
388          # Get the categories, and order them alphabetically. 
389          categories = builderList.keys() 
390          categories.sort() 
391   
392          slaves = {} 
393   
394          # For each category, we display each builder. 
395          for category in categories: 
396              slaves[category] = [] 
397              # For each builder in this category, we set the build info and we 
398              # display the box. 
399              for builder in builderList[category]: 
400                  s = {} 
401                  s["color"] = "notstarted" 
402                  s["title"] = builder 
403                  s["url"] = "./builders/%s" % urllib.quote(builder) 
404                  state, builds = status.getBuilder(builder).getState() 
405                  # Check if it's offline, if so, the box is purple. 
406                  if state == "offline": 
407                      s["color"] = "offline" 
408                  else: 
409                      # If not offline, then display the result of the last 
410                      # finished build. 
411                      build = self.getHeadBuild(status.getBuilder(builder)) 
412                      while build and not build.isFinished(): 
413                          build = build.getPreviousBuild() 
414   
415                      if build: 
416                          s["color"] = getResultsClass(build.getResults(), None, 
417                                                        False) 
418   
419                  slaves[category].append(s) 
420   
421          return slaves 
422   
423 -    def displayStatusLine(self, builderList, allBuilds, revision, debugInfo): 
424          """Display the boxes that represent the status of each builder in the 
425          first build "revision" was in. Returns an HTML list of errors that 
426          happened during these builds.""" 
427   
428          details = [] 
429          nbSlaves = 0 
430          for category in builderList: 
431              nbSlaves += len(builderList[category]) 
432   
433          # Sort the categories. 
434          categories = builderList.keys() 
435          categories.sort() 
436           
437          builds = {} 
438     
439          # Display the boxes by category group. 
440          for category in categories: 
441     
442              builds[category] = [] 
443               
444              # Display the boxes for each builder in this category. 
445              for builder in builderList[category]: 
446                  introducedIn = None 
447                  firstNotIn = None 
448   
449                  # Find the first build that does not include the revision. 
450                  for build in allBuilds[builder]: 
451                      if self.comparator.isRevisionEarlier(build, revision): 
452                          firstNotIn = build 
453                          break 
454                      else: 
455                          introducedIn = build 
456                           
457                  # Get the results of the first build with the revision, and the 
458                  # first build that does not include the revision. 
459                  results = None 
460                  previousResults = None 
461                  if introducedIn: 
462                      results = introducedIn.results 
463                  if firstNotIn: 
464                      previousResults = firstNotIn.results 
465   
466                  isRunning = False 
467                  if introducedIn and not introducedIn.isFinished: 
468                      isRunning = True 
469   
470                  url = "./waterfall" 
471                  title = builder 
472                  tag = "" 
473                  current_details = {} 
474                  if introducedIn: 
475                      current_details = introducedIn.details or "" 
476                      url = "./buildstatus?builder=%s&number=%s" % (urllib.quote(builder), 
477                                                                    introducedIn.number) 
478                      title += " " 
479                      title += urllib.quote(' '.join(introducedIn.text), ' \n\\/:') 
480   
481                      builderStrip = builder.replace(' ', '') 
482                      builderStrip = builderStrip.replace('(', '') 
483                      builderStrip = builderStrip.replace(')', '') 
484                      builderStrip = builderStrip.replace('.', '') 
485                      tag = "Tag%s%s" % (builderStrip, introducedIn.number) 
486   
487                  if isRunning: 
488                      title += ' ETA: %ds' % (introducedIn.eta or 0) 
489                       
490                  resultsClass = getResultsClass(results, previousResults, isRunning) 
491   
492                  b = {}                 
493                  b["url"] = url 
494                  b["title"] = title 
495                  b["color"] = resultsClass 
496                  b["tag"] = tag 
497   
498                  builds[category].append(b) 
499   
500                  # If the box is red, we add the explaination in the details 
501                  # section. 
502                  if current_details and resultsClass == "failure": 
503                      details.append(current_details) 
504   
505          return (builds, details) 
506   
507 -    def filterRevisions(self, revisions, filter=None, max_revs=None): 
508          """Filter a set of revisions based on any number of filter criteria. 
509          If specified, filter should be a dict with keys corresponding to 
510          revision attributes, and values of 1+ strings""" 
511          if not filter: 
512              if max_revs is None: 
513                  for rev in reversed(revisions): 
514                      yield DevRevision(rev) 
515              else: 
516                  for index,rev in enumerate(reversed(revisions)): 
517                      if index >= max_revs: 
518                          break 
519                      yield DevRevision(rev) 
520          else: 
521              for index, rev in enumerate(reversed(revisions)): 
522                  if max_revs and index >= max_revs: 
523                      break 
524                  try: 
525                      for field,acceptable in filter.iteritems(): 
526                          if not hasattr(rev, field): 
527                              raise DoesNotPassFilter 
528                          if type(acceptable) in (str, unicode): 
529                              if getattr(rev, field) != acceptable: 
530                                  raise DoesNotPassFilter 
531                          elif type(acceptable) in (list, tuple, set): 
532                              if getattr(rev, field) not in acceptable: 
533                                  raise DoesNotPassFilter 
534                      yield DevRevision(rev) 
535                  except DoesNotPassFilter: 
536                      pass 
537   
538 -    def displayPage(self, request, status, builderList, allBuilds, revisions, 
539                      categories, repository, branch, debugInfo): 
540          """Display the console page.""" 
541          # Build the main template directory with all the informations we have. 
542          subs = dict() 
543          subs["branch"] = branch or 'trunk' 
544          subs["repository"] = repository 
545          if categories: 
546              subs["categories"] = ' '.join(categories) 
547          subs["time"] = time.strftime("%a %d %b %Y %H:%M:%S", 
548                                       time.localtime(util.now())) 
549          subs["debugInfo"] = debugInfo 
550          subs["ANYBRANCH"] = ANYBRANCH 
551   
552          if builderList: 
553              subs["categories"] = self.displayCategories(builderList, debugInfo) 
554              subs['slaves'] = self.displaySlaveLine(status, builderList, debugInfo) 
555          else: 
556              subs["categories"] = [] 
557   
558          subs['revisions'] = [] 
559   
560          # For each revision we show one line 
561          for revision in revisions: 
562              r = {} 
563               
564              # Fill the dictionary with this new information 
565              r['id'] = revision.revision 
566              r['link'] = revision.revlink  
567              r['who'] = revision.who 
568              r['date'] = revision.date 
569              r['comments'] = revision.comments 
570              r['repository'] = revision.repository 
571              r['project'] = revision.project 
572   
573              # Display the status for all builders. 
574              (builds, details) = self.displayStatusLine(builderList, 
575                                              allBuilds, 
576                                              revision, 
577                                              debugInfo) 
578              r['builds'] = builds 
579              r['details'] = details 
580   
581              # Calculate the td span for the comment and the details. 
582              r["span"] = len(builderList) + 2             
583   
584              subs['revisions'].append(r) 
585   
586          # 
587          # Display the footer of the page. 
588          # 
589          debugInfo["load_time"] = time.time() - debugInfo["load_time"] 
590          return subs 
591   
592   
593 -    def content(self, request, cxt): 
594          "This method builds the main console view display." 
595   
596          reload_time = None 
597          # Check if there was an arg. Don't let people reload faster than 
598          # every 15 seconds. 0 means no reload. 
599          if "reload" in request.args: 
600              try: 
601                  reload_time = int(request.args["reload"][0]) 
602                  if reload_time != 0: 
603                      reload_time = max(reload_time, 15) 
604              except ValueError: 
605                  pass 
606   
607          request.setHeader('Cache-Control', 'no-cache') 
608   
609          # Sets the default reload time to 60 seconds. 
610          if not reload_time: 
611              reload_time = 60 
612   
613          # Append the tag to refresh the page.  
614          if reload_time is not None and reload_time != 0: 
615              cxt['refresh'] = reload_time 
616   
617          # Debug information to display at the end of the page. 
618          debugInfo = cxt['debuginfo'] = dict() 
619          debugInfo["load_time"] = time.time() 
620   
621          # get url parameters 
622          # Categories to show information for. 
623          categories = request.args.get("category", []) 
624          # List of all builders to show on the page. 
625          builders = request.args.get("builder", []) 
626          # Repo used to filter the changes shown. 
627          repository = request.args.get("repository", [None])[0] 
628          # Branch used to filter the changes shown. 
629          branch = request.args.get("branch", [ANYBRANCH])[0] 
630          # List of all the committers name to display on the page. 
631          devName = request.args.get("name", []) 
632   
633          # and the data we want to render 
634          status = self.getStatus(request) 
635   
636          # Get all revisions we can find. 
637          source = self.getChangeManager(request) 
638          allChanges = self.getAllChanges(source, status, debugInfo) 
639   
640          debugInfo["source_all"] = len(allChanges) 
641   
642          # Keep only the revisions we care about. 
643          # By default we process the last 40 revisions. 
644          # If a dev name is passed, we look for the changes by this person in the 
645          # last 80 revisions. 
646          numRevs = int(request.args.get("revs", [40])[0]) 
647          if devName: 
648              numRevs *= 2 
649          numBuilds = numRevs 
650   
651          revFilter = {} 
652          if branch != ANYBRANCH: 
653              revFilter['branch'] = branch 
654          if devName: 
655              revFilter['who'] = devName 
656          if repository: 
657              revFilter['repository'] = repository 
658          revisions = list(self.filterRevisions(allChanges, max_revs=numRevs, filter=revFilter)) 
659          debugInfo["revision_final"] = len(revisions) 
660   
661          # Fetch all the builds for all builders until we get the next build 
662          # after lastRevision. 
663          builderList = None 
664          allBuilds = None 
665          if revisions: 
666              lastRevision = revisions[len(revisions) - 1].revision 
667              debugInfo["last_revision"] = lastRevision 
668   
669              (builderList, allBuilds) = self.getAllBuildsForRevision(status, 
670                                                  request, 
671                                                  lastRevision, 
672                                                  numBuilds, 
673                                                  categories, 
674                                                  builders, 
675                                                  debugInfo) 
676   
677          debugInfo["added_blocks"] = 0 
678   
679          cxt.update(self.displayPage(request, status, builderList, allBuilds, 
680                                      revisions, categories, repository, branch, debugInfo)) 
681   
682          template = request.site.buildbot_service.templates.get_template("console.html") 
683          data = template.render(cxt) 
684          return data 
685   
686 -class RevisionComparator(object): 
687      """Used for comparing between revisions, as some 
688      VCS use a plain counter for revisions (like SVN) 
689      while others use different concepts (see Git). 
690      """ 
691       
692      # TODO (avivby): Should this be a zope interface? 
693       
694 -    def isRevisionEarlier(self, first_change, second_change): 
695          """Used for comparing 2 changes""" 
696          raise NotImplementedError 
697   
698 -    def isValidRevision(self, revision): 
699          """Checks whether the revision seems like a VCS revision""" 
700          raise NotImplementedError 
701   
702 -    def getSortingKey(self): 
703          raise NotImplementedError 
704       
705 -class TimeRevisionComparator(RevisionComparator): 
706 -    def isRevisionEarlier(self, first, second): 
707          return first.when < second.when 
708   
709 -    def isValidRevision(self, revision): 
710          return True # No general way of determining 
711   
712 -    def getSortingKey(self): 
713          return operator.attrgetter('when') 
714   
715 -class IntegerRevisionComparator(RevisionComparator): 
716 -    def isRevisionEarlier(self, first, second): 
717          try: 
718              return int(first.revision) < int(second.revision) 
719          except TypeError: 
720              return False 
721   
722 -    def isValidRevision(self, revision): 
723          try: 
724              int(revision) 
725              return True 
726          except: 
727              return False 
728   
729 -    def getSortingKey(self): 
730          return operator.attrgetter('revision') 
731