Source Code for Module buildbot.steps.python_twisted

  1  # This file is part of Buildbot.  Buildbot is free software: you can 
  2  # redistribute it and/or modify it under the terms of the GNU General Public 
  3  # License as published by the Free Software Foundation, version 2. 
  4  # 
  5  # This program is distributed in the hope that it will be useful, but WITHOUT 
  6  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
  7  # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more 
  8  # details. 
  9  # 
 10  # You should have received a copy of the GNU General Public License along with 
 11  # this program; if not, write to the Free Software Foundation, Inc., 51 
 12  # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 
 13  # 
 14  # Copyright Buildbot Team Members 
 15   
 16   
 17  from twisted.python import log 
 18   
 19  from buildbot.status import testresult 
 20  from buildbot.status.results import SUCCESS, FAILURE, WARNINGS, SKIPPED 
 21  from buildbot.process.buildstep import LogLineObserver, OutputProgressObserver 
 22  from buildbot.steps.shell import ShellCommand 
 23   
 24  try: 
 25      import cStringIO 
 26      StringIO = cStringIO 
 27  except ImportError: 
 28      import StringIO 
 29  import re 
 30   
 31  # BuildSteps that are specific to the Twisted source tree 
 32   
 33 -class HLint(ShellCommand): 
 34      """I run a 'lint' checker over a set of .xhtml files. Any deviations 
 35      from recommended style is flagged and put in the output log. 
 36   
 37      This step looks at .changes in the parent Build to extract a list of 
 38      Lore XHTML files to check.""" 
 39   
 40      name = "hlint" 
 41      description = ["running", "hlint"] 
 42      descriptionDone = ["hlint"] 
 43      warnOnWarnings = True 
 44      warnOnFailure = True 
 45      # TODO: track time, but not output 
 46      warnings = 0 
 47   
 48 -    def __init__(self, python=None, **kwargs): 
 49          ShellCommand.__init__(self, **kwargs) 
 50          self.python = python 
 51   
 52 -    def start(self): 
 53          # create the command 
 54          htmlFiles = {} 
 55          for f in self.build.allFiles(): 
 56              if f.endswith(".xhtml") and not f.startswith("sandbox/"): 
 57                  htmlFiles[f] = 1 
 58          # remove duplicates 
 59          hlintTargets = htmlFiles.keys() 
 60          hlintTargets.sort() 
 61          if not hlintTargets: 
 62              return SKIPPED 
 63          self.hlintFiles = hlintTargets 
 64          c = [] 
 65          if self.python: 
 66              c.append(self.python) 
 67          c += ["bin/lore", "-p", "--output", "lint"] + self.hlintFiles 
 68          self.setCommand(c) 
 69   
 70          # add an extra log file to show the .html files we're checking 
 71          self.addCompleteLog("files", "\n".join(self.hlintFiles)+"\n") 
 72   
 73          ShellCommand.start(self) 
 74   
 75 -    def commandComplete(self, cmd): 
 76          # TODO: remove the 'files' file (a list of .xhtml files that were 
 77          # submitted to hlint) because it is available in the logfile and 
 78          # mostly exists to give the user an idea of how long the step will 
 79          # take anyway). 
 80          lines = cmd.logs['stdio'].getText().split("\n") 
 81          warningLines = filter(lambda line:':' in line, lines) 
 82          if warningLines: 
 83              self.addCompleteLog("warnings", "".join(warningLines)) 
 84          warnings = len(warningLines) 
 85          self.warnings = warnings 
 86   
 87 -    def evaluateCommand(self, cmd): 
 88          # warnings are in stdout, rc is always 0, unless the tools break 
 89          if cmd.didFail(): 
 90              return FAILURE 
 91          if self.warnings: 
 92              return WARNINGS 
 93          return SUCCESS 
 94   
 95 -    def getText2(self, cmd, results): 
 96          if cmd.didFail(): 
 97              return ["hlint"] 
 98          return ["%d hlin%s" % (self.warnings, 
 99                                 self.warnings == 1 and 't' or 'ts')] 
100   
101 -def countFailedTests(output): 
102      # start scanning 10kb from the end, because there might be a few kb of 
103      # import exception tracebacks between the total/time line and the errors 
104      # line 
105      chunk = output[-10000:] 
106      lines = chunk.split("\n") 
107      lines.pop() # blank line at end 
108      # lines[-3] is "Ran NN tests in 0.242s" 
109      # lines[-2] is blank 
110      # lines[-1] is 'OK' or 'FAILED (failures=1, errors=12)' 
111      #  or 'FAILED (failures=1)' 
112      #  or "PASSED (skips=N, successes=N)"  (for Twisted-2.0) 
113      # there might be other lines dumped here. Scan all the lines. 
114      res = {'total': None, 
115             'failures': 0, 
116             'errors': 0, 
117             'skips': 0, 
118             'expectedFailures': 0, 
119             'unexpectedSuccesses': 0, 
120             } 
121      for l in lines: 
122          out = re.search(r'Ran (\d+) tests', l) 
123          if out: 
124              res['total'] = int(out.group(1)) 
125          if (l.startswith("OK") or 
126              l.startswith("FAILED ") or 
127              l.startswith("PASSED")): 
128              # the extra space on FAILED_ is to distinguish the overall 
129              # status from an individual test which failed. The lack of a 
130              # space on the OK is because it may be printed without any 
131              # additional text (if there are no skips,etc) 
132              out = re.search(r'failures=(\d+)', l) 
133              if out: res['failures'] = int(out.group(1)) 
134              out = re.search(r'errors=(\d+)', l) 
135              if out: res['errors'] = int(out.group(1)) 
136              out = re.search(r'skips=(\d+)', l) 
137              if out: res['skips'] = int(out.group(1)) 
138              out = re.search(r'expectedFailures=(\d+)', l) 
139              if out: res['expectedFailures'] = int(out.group(1)) 
140              out = re.search(r'unexpectedSuccesses=(\d+)', l) 
141              if out: res['unexpectedSuccesses'] = int(out.group(1)) 
142              # successes= is a Twisted-2.0 addition, and is not currently used 
143              out = re.search(r'successes=(\d+)', l) 
144              if out: res['successes'] = int(out.group(1)) 
145   
146      return res 
147   
148   
149 -class TrialTestCaseCounter(LogLineObserver): 
150      _line_re = re.compile(r'^(?:Doctest: )?([\w\.]+) \.\.\. \[([^\]]+)\]$') 
151      numTests = 0 
152      finished = False 
153   
154 -    def outLineReceived(self, line): 
155          # different versions of Twisted emit different per-test lines with 
156          # the bwverbose reporter. 
157          #  2.0.0: testSlave (buildbot.test.test_runner.Create) ... [OK] 
158          #  2.1.0: buildbot.test.test_runner.Create.testSlave ... [OK] 
159          #  2.4.0: buildbot.test.test_runner.Create.testSlave ... [OK] 
160          # Let's just handle the most recent version, since it's the easiest. 
161          # Note that doctests create lines line this: 
162          #  Doctest: viff.field.GF ... [OK] 
163   
164          if self.finished: 
165              return 
166          if line.startswith("=" * 40): 
167              self.finished = True 
168              return 
169   
170          m = self._line_re.search(line.strip()) 
171          if m: 
172              testname, result = m.groups() 
173              self.numTests += 1 
174              self.step.setProgress('tests', self.numTests) 
175   
176   
177  UNSPECIFIED=() # since None is a valid choice 
178   
179 -class Trial(ShellCommand): 
180      """ 
181      There are some class attributes which may be usefully overridden 
182      by subclasses. 'trialMode' and 'trialArgs' can influence the trial 
183      command line. 
184      """ 
185   
186      name = "trial" 
187      progressMetrics = ('output', 'tests', 'test.log') 
188      # note: the slash only works on unix buildslaves, of course, but we have 
189      # no way to know what the buildslave uses as a separator.  
190      # TODO: figure out something clever. 
191      logfiles = {"test.log": "_trial_temp/test.log"} 
192      # we use test.log to track Progress at the end of __init__() 
193   
194      renderables = ['tests'] 
195      flunkOnFailure = True 
196      python = None 
197      trial = "trial" 
198      trialMode = ["--reporter=bwverbose"] # requires Twisted-2.1.0 or newer 
199      # for Twisted-2.0.0 or 1.3.0, use ["-o"] instead 
200      trialArgs = [] 
201      testpath = UNSPECIFIED # required (but can be None) 
202      testChanges = False # TODO: needs better name 
203      recurse = False 
204      reactor = None 
205      randomly = False 
206      tests = None # required 
207   
208 -    def __init__(self, reactor=UNSPECIFIED, python=None, trial=None, 
209                   testpath=UNSPECIFIED, 
210                   tests=None, testChanges=None, 
211                   recurse=None, randomly=None, 
212                   trialMode=None, trialArgs=None, 
213                   **kwargs): 
214          """ 
215          @type  testpath: string 
216          @param testpath: use in PYTHONPATH when running the tests. If 
217                           None, do not set PYTHONPATH. Setting this to '.' will 
218                           cause the source files to be used in-place. 
219   
220          @type  python: string (without spaces) or list 
221          @param python: which python executable to use. Will form the start of 
222                         the argv array that will launch trial. If you use this, 
223                         you should set 'trial' to an explicit path (like 
224                         /usr/bin/trial or ./bin/trial). Defaults to None, which 
225                         leaves it out entirely (running 'trial args' instead of 
226                         'python ./bin/trial args'). Likely values are 'python', 
227                         ['python2.2'], ['python', '-Wall'], etc. 
228   
229          @type  trial: string 
230          @param trial: which 'trial' executable to run. 
231                        Defaults to 'trial', which will cause $PATH to be 
232                        searched and probably find /usr/bin/trial . If you set 
233                        'python', this should be set to an explicit path (because 
234                        'python2.3 trial' will not work). 
235   
236          @type trialMode: list of strings 
237          @param trialMode: a list of arguments to pass to trial, specifically 
238                            to set the reporting mode. This defaults to ['-to'] 
239                            which means 'verbose colorless output' to the trial 
240                            that comes with Twisted-2.0.x and at least -2.1.0 . 
241                            Newer versions of Twisted may come with a trial 
242                            that prefers ['--reporter=bwverbose']. 
243   
244          @type trialArgs: list of strings 
245          @param trialArgs: a list of arguments to pass to trial, available to 
246                            turn on any extra flags you like. Defaults to []. 
247   
248          @type  tests: list of strings 
249          @param tests: a list of test modules to run, like 
250                        ['twisted.test.test_defer', 'twisted.test.test_process']. 
251                        If this is a string, it will be converted into a one-item 
252                        list. 
253   
254          @type  testChanges: boolean 
255          @param testChanges: if True, ignore the 'tests' parameter and instead 
256                              ask the Build for all the files that make up the 
257                              Changes going into this build. Pass these filenames 
258                              to trial and ask it to look for test-case-name 
259                              tags, running just the tests necessary to cover the 
260                              changes. 
261   
262          @type  recurse: boolean 
263          @param recurse: If True, pass the --recurse option to trial, allowing 
264                          test cases to be found in deeper subdirectories of the 
265                          modules listed in 'tests'. This does not appear to be 
266                          necessary when using testChanges. 
267   
268          @type  reactor: string 
269          @param reactor: which reactor to use, like 'gtk' or 'java'. If not 
270                          provided, the Twisted's usual platform-dependent 
271                          default is used. 
272   
273          @type  randomly: boolean 
274          @param randomly: if True, add the --random=0 argument, which instructs 
275                           trial to run the unit tests in a random order each 
276                           time. This occasionally catches problems that might be 
277                           masked when one module always runs before another 
278                           (like failing to make registerAdapter calls before 
279                           lookups are done). 
280   
281          @type  kwargs: dict 
282          @param kwargs: parameters. The following parameters are inherited from 
283                         L{ShellCommand} and may be useful to set: workdir, 
284                         haltOnFailure, flunkOnWarnings, flunkOnFailure, 
285                         warnOnWarnings, warnOnFailure, want_stdout, want_stderr, 
286                         timeout. 
287          """ 
288          ShellCommand.__init__(self, **kwargs) 
289   
290          if python: 
291              self.python = python 
292          if self.python is not None: 
293              if type(self.python) is str: 
294                  self.python = [self.python] 
295              for s in self.python: 
296                  if " " in s: 
297                      # this is not strictly an error, but I suspect more 
298                      # people will accidentally try to use python="python2.3 
299                      # -Wall" than will use embedded spaces in a python flag 
300                      log.msg("python= component '%s' has spaces") 
301                      log.msg("To add -Wall, use python=['python', '-Wall']") 
302                      why = "python= value has spaces, probably an error" 
303                      raise ValueError(why) 
304   
305          if trial: 
306              self.trial = trial 
307          if " " in self.trial: 
308              raise ValueError("trial= value has spaces") 
309          if trialMode is not None: 
310              self.trialMode = trialMode 
311          if trialArgs is not None: 
312              self.trialArgs = trialArgs 
313   
314          if testpath is not UNSPECIFIED: 
315              self.testpath = testpath 
316          if self.testpath is UNSPECIFIED: 
317              raise ValueError("You must specify testpath= (it can be None)") 
318          assert isinstance(self.testpath, str) or self.testpath is None 
319   
320          if reactor is not UNSPECIFIED: 
321              self.reactor = reactor 
322   
323          if tests is not None: 
324              self.tests = tests 
325          if type(self.tests) is str: 
326              self.tests = [self.tests] 
327          if testChanges is not None: 
328              self.testChanges = testChanges 
329              #self.recurse = True  # not sure this is necessary 
330   
331          if not self.testChanges and self.tests is None: 
332              raise ValueError("Must either set testChanges= or provide tests=") 
333   
334          if recurse is not None: 
335              self.recurse = recurse 
336          if randomly is not None: 
337              self.randomly = randomly 
338   
339          # build up most of the command, then stash it until start() 
340          command = [] 
341          if self.python: 
342              command.extend(self.python) 
343          command.append(self.trial) 
344          command.extend(self.trialMode) 
345          if self.recurse: 
346              command.append("--recurse") 
347          if self.reactor: 
348              command.append("--reactor=%s" % reactor) 
349          if self.randomly: 
350              command.append("--random=0") 
351          command.extend(self.trialArgs) 
352          self.command = command 
353   
354          if self.reactor: 
355              self.description = ["testing", "(%s)" % self.reactor] 
356              self.descriptionDone = ["tests"] 
357              # commandComplete adds (reactorname) to self.text 
358          else: 
359              self.description = ["testing"] 
360              self.descriptionDone = ["tests"] 
361   
362          # this counter will feed Progress along the 'test cases' metric 
363          self.addLogObserver('stdio', TrialTestCaseCounter()) 
364          # this one just measures bytes of output in _trial_temp/test.log 
365          self.addLogObserver('test.log', OutputProgressObserver('test.log')) 
366   
367 -    def setupEnvironment(self, cmd): 
368          ShellCommand.setupEnvironment(self, cmd) 
369          if self.testpath != None: 
370              e = cmd.args['env'] 
371              if e is None: 
372                  cmd.args['env'] = {'PYTHONPATH': self.testpath} 
373              else: 
374                  #this bit produces a list, which can be used 
375                  #by buildslave.runprocess.RunProcess 
376                  ppath = e.get('PYTHONPATH', self.testpath) 
377                  if isinstance(ppath, str): 
378                      ppath = [ppath] 
379                  if self.testpath not in ppath: 
380                      ppath.insert(0, self.testpath) 
381                  e['PYTHONPATH'] = ppath 
382   
383 -    def start(self): 
384          # now that self.build.allFiles() is nailed down, finish building the 
385          # command 
386          if self.testChanges: 
387              for f in self.build.allFiles(): 
388                  if f.endswith(".py"): 
389                      self.command.append("--testmodule=%s" % f) 
390          else: 
391              self.command.extend(self.tests) 
392          log.msg("Trial.start: command is", self.command) 
393   
394          ShellCommand.start(self) 
395   
396   
397 -    def commandComplete(self, cmd): 
398          # figure out all status, then let the various hook functions return 
399          # different pieces of it 
400   
401          # 'cmd' is the original trial command, so cmd.logs['stdio'] is the 
402          # trial output. We don't have access to test.log from here. 
403          output = cmd.logs['stdio'].getText() 
404          counts = countFailedTests(output) 
405   
406          total = counts['total'] 
407          failures, errors = counts['failures'], counts['errors'] 
408          parsed = (total != None) 
409          text = [] 
410          text2 = "" 
411   
412          if not cmd.didFail(): 
413              if parsed: 
414                  results = SUCCESS 
415                  if total: 
416                      text += ["%d %s" % \ 
417                               (total, 
418                                total == 1 and "test" or "tests"), 
419                               "passed"] 
420                  else: 
421                      text += ["no tests", "run"] 
422              else: 
423                  results = FAILURE 
424                  text += ["testlog", "unparseable"] 
425                  text2 = "tests" 
426          else: 
427              # something failed 
428              results = FAILURE 
429              if parsed: 
430                  text.append("tests") 
431                  if failures: 
432                      text.append("%d %s" % \ 
433                                  (failures, 
434                                   failures == 1 and "failure" or "failures")) 
435                  if errors: 
436                      text.append("%d %s" % \ 
437                                  (errors, 
438                                   errors == 1 and "error" or "errors")) 
439                  count = failures + errors 
440                  text2 = "%d tes%s" % (count, (count == 1 and 't' or 'ts')) 
441              else: 
442                  text += ["tests", "failed"] 
443                  text2 = "tests" 
444   
445          if counts['skips']: 
446              text.append("%d %s" %  \ 
447                          (counts['skips'], 
448                           counts['skips'] == 1 and "skip" or "skips")) 
449          if counts['expectedFailures']: 
450              text.append("%d %s" %  \ 
451                          (counts['expectedFailures'], 
452                           counts['expectedFailures'] == 1 and "todo" 
453                           or "todos")) 
454              if 0: # TODO 
455                  results = WARNINGS 
456                  if not text2: 
457                      text2 = "todo" 
458   
459          if 0: 
460              # ignore unexpectedSuccesses for now, but it should really mark 
461              # the build WARNING 
462              if counts['unexpectedSuccesses']: 
463                  text.append("%d surprises" % counts['unexpectedSuccesses']) 
464                  results = WARNINGS 
465                  if not text2: 
466                      text2 = "tests" 
467   
468          if self.reactor: 
469              text.append(self.rtext('(%s)')) 
470              if text2: 
471                  text2 = "%s %s" % (text2, self.rtext('(%s)')) 
472   
473          self.results = results 
474          self.text = text 
475          self.text2 = [text2] 
476   
477   
478 -    def rtext(self, fmt='%s'): 
479          if self.reactor: 
480              rtext = fmt % self.reactor 
481              return rtext.replace("reactor", "") 
482          return "" 
483   
484 -    def addTestResult(self, testname, results, text, tlog): 
485          if self.reactor is not None: 
486              testname = (self.reactor,) + testname 
487          tr = testresult.TestResult(testname, results, text, logs={'log': tlog}) 
488          #self.step_status.build.addTestResult(tr) 
489          self.build.build_status.addTestResult(tr) 
490   
491 -    def createSummary(self, loog): 
492          output = loog.getText() 
493          problems = "" 
494          sio = StringIO.StringIO(output) 
495          warnings = {} 
496          while 1: 
497              line = sio.readline() 
498              if line == "": 
499                  break 
500              if line.find(" exceptions.DeprecationWarning: ") != -1: 
501                  # no source 
502                  warning = line # TODO: consider stripping basedir prefix here 
503                  warnings[warning] = warnings.get(warning, 0) + 1 
504              elif (line.find(" DeprecationWarning: ") != -1 or 
505                  line.find(" UserWarning: ") != -1): 
506                  # next line is the source 
507                  warning = line + sio.readline() 
508                  warnings[warning] = warnings.get(warning, 0) + 1 
509              elif line.find("Warning: ") != -1: 
510                  warning = line 
511                  warnings[warning] = warnings.get(warning, 0) + 1 
512   
513              if line.find("=" * 60) == 0 or line.find("-" * 60) == 0: 
514                  problems += line 
515                  problems += sio.read() 
516                  break 
517   
518          if problems: 
519              self.addCompleteLog("problems", problems) 
520              # now parse the problems for per-test results 
521              pio = StringIO.StringIO(problems) 
522              pio.readline() # eat the first separator line 
523              testname = None 
524              done = False 
525              while not done: 
526                  while 1: 
527                      line = pio.readline() 
528                      if line == "": 
529                          done = True 
530                          break 
531                      if line.find("=" * 60) == 0: 
532                          break 
533                      if line.find("-" * 60) == 0: 
534                          # the last case has --- as a separator before the 
535                          # summary counts are printed 
536                          done = True 
537                          break 
538                      if testname is None: 
539                          # the first line after the === is like: 
540  # EXPECTED FAILURE: testLackOfTB (twisted.test.test_failure.FailureTestCase) 
541  # SKIPPED: testRETR (twisted.test.test_ftp.TestFTPServer) 
542  # FAILURE: testBatchFile (twisted.conch.test.test_sftp.TestOurServerBatchFile) 
543                          r = re.search(r'^([^:]+): (\w+) \(([\w\.]+)\)', line) 
544                          if not r: 
545                              # TODO: cleanup, if there are no problems, 
546                              # we hit here 
547                              continue 
548                          result, name, case = r.groups() 
549                          testname = tuple(case.split(".") + [name]) 
550                          results = {'SKIPPED': SKIPPED, 
551                                     'EXPECTED FAILURE': SUCCESS, 
552                                     'UNEXPECTED SUCCESS': WARNINGS, 
553                                     'FAILURE': FAILURE, 
554                                     'ERROR': FAILURE, 
555                                     'SUCCESS': SUCCESS, # not reported 
556                                     }.get(result, WARNINGS) 
557                          text = result.lower().split() 
558                          loog = line 
559                          # the next line is all dashes 
560                          loog += pio.readline() 
561                      else: 
562                          # the rest goes into the log 
563                          loog += line 
564                  if testname: 
565                      self.addTestResult(testname, results, text, loog) 
566                      testname = None 
567   
568          if warnings: 
569              lines = warnings.keys() 
570              lines.sort() 
571              self.addCompleteLog("warnings", "".join(lines)) 
572   
573 -    def evaluateCommand(self, cmd): 
574          return self.results 
575   
576 -    def getText(self, cmd, results): 
577          return self.text 
578 -    def getText2(self, cmd, results): 
579          return self.text2 
580   
581   
582 -class RemovePYCs(ShellCommand): 
583      name = "remove-.pyc" 
584      command = ['find', '.', '-name', '*.pyc', '-exec', 'rm', '{}', ';'] 
585      description = ["removing", ".pyc", "files"] 
586      descriptionDone = ["remove", ".pycs"] 
587