Source Code for Module buildbot.steps.python_twisted

  1  # This file is part of Buildbot.  Buildbot is free software: you can 
  2  # redistribute it and/or modify it under the terms of the GNU General Public 
  3  # License as published by the Free Software Foundation, version 2. 
  4  # 
  5  # This program is distributed in the hope that it will be useful, but WITHOUT 
  6  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
  7  # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more 
  8  # details. 
  9  # 
 10  # You should have received a copy of the GNU General Public License along with 
 11  # this program; if not, write to the Free Software Foundation, Inc., 51 
 12  # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 
 13  # 
 14  # Copyright Buildbot Team Members 
 15   
 16   
 17  from twisted.python import log 
 18   
 19  from buildbot.status import testresult 
 20  from buildbot.status.results import SUCCESS, FAILURE, WARNINGS, SKIPPED 
 21  from buildbot.process.buildstep import LogLineObserver, OutputProgressObserver 
 22  from buildbot.steps.shell import ShellCommand 
 23   
 24  try: 
 25      import cStringIO 
 26      StringIO = cStringIO 
 27  except ImportError: 
 28      import StringIO 
 29  import re 
 30   
 31  # BuildSteps that are specific to the Twisted source tree 
 32   
 33 -class HLint(ShellCommand): 
 34      """I run a 'lint' checker over a set of .xhtml files. Any deviations 
 35      from recommended style is flagged and put in the output log. 
 36   
 37      This step looks at .changes in the parent Build to extract a list of 
 38      Lore XHTML files to check.""" 
 39   
 40      name = "hlint" 
 41      description = ["running", "hlint"] 
 42      descriptionDone = ["hlint"] 
 43      warnOnWarnings = True 
 44      warnOnFailure = True 
 45      # TODO: track time, but not output 
 46      warnings = 0 
 47   
 48 -    def __init__(self, python=None, **kwargs): 
 49          ShellCommand.__init__(self, **kwargs) 
 50          self.addFactoryArguments(python=python) 
 51          self.python = python 
 52   
 53 -    def start(self): 
 54          # create the command 
 55          htmlFiles = {} 
 56          for f in self.build.allFiles(): 
 57              if f.endswith(".xhtml") and not f.startswith("sandbox/"): 
 58                  htmlFiles[f] = 1 
 59          # remove duplicates 
 60          hlintTargets = htmlFiles.keys() 
 61          hlintTargets.sort() 
 62          if not hlintTargets: 
 63              return SKIPPED 
 64          self.hlintFiles = hlintTargets 
 65          c = [] 
 66          if self.python: 
 67              c.append(self.python) 
 68          c += ["bin/lore", "-p", "--output", "lint"] + self.hlintFiles 
 69          self.setCommand(c) 
 70   
 71          # add an extra log file to show the .html files we're checking 
 72          self.addCompleteLog("files", "\n".join(self.hlintFiles)+"\n") 
 73   
 74          ShellCommand.start(self) 
 75   
 76 -    def commandComplete(self, cmd): 
 77          # TODO: remove the 'files' file (a list of .xhtml files that were 
 78          # submitted to hlint) because it is available in the logfile and 
 79          # mostly exists to give the user an idea of how long the step will 
 80          # take anyway). 
 81          lines = cmd.logs['stdio'].getText().split("\n") 
 82          warningLines = filter(lambda line:':' in line, lines) 
 83          if warningLines: 
 84              self.addCompleteLog("warnings", "".join(warningLines)) 
 85          warnings = len(warningLines) 
 86          self.warnings = warnings 
 87   
 88 -    def evaluateCommand(self, cmd): 
 89          # warnings are in stdout, rc is always 0, unless the tools break 
 90          if cmd.rc != 0: 
 91              return FAILURE 
 92          if self.warnings: 
 93              return WARNINGS 
 94          return SUCCESS 
 95   
 96 -    def getText2(self, cmd, results): 
 97          if cmd.rc != 0: 
 98              return ["hlint"] 
 99          return ["%d hlin%s" % (self.warnings, 
100                                 self.warnings == 1 and 't' or 'ts')] 
101   
102 -def countFailedTests(output): 
103      # start scanning 10kb from the end, because there might be a few kb of 
104      # import exception tracebacks between the total/time line and the errors 
105      # line 
106      chunk = output[-10000:] 
107      lines = chunk.split("\n") 
108      lines.pop() # blank line at end 
109      # lines[-3] is "Ran NN tests in 0.242s" 
110      # lines[-2] is blank 
111      # lines[-1] is 'OK' or 'FAILED (failures=1, errors=12)' 
112      #  or 'FAILED (failures=1)' 
113      #  or "PASSED (skips=N, successes=N)"  (for Twisted-2.0) 
114      # there might be other lines dumped here. Scan all the lines. 
115      res = {'total': None, 
116             'failures': 0, 
117             'errors': 0, 
118             'skips': 0, 
119             'expectedFailures': 0, 
120             'unexpectedSuccesses': 0, 
121             } 
122      for l in lines: 
123          out = re.search(r'Ran (\d+) tests', l) 
124          if out: 
125              res['total'] = int(out.group(1)) 
126          if (l.startswith("OK") or 
127              l.startswith("FAILED ") or 
128              l.startswith("PASSED")): 
129              # the extra space on FAILED_ is to distinguish the overall 
130              # status from an individual test which failed. The lack of a 
131              # space on the OK is because it may be printed without any 
132              # additional text (if there are no skips,etc) 
133              out = re.search(r'failures=(\d+)', l) 
134              if out: res['failures'] = int(out.group(1)) 
135              out = re.search(r'errors=(\d+)', l) 
136              if out: res['errors'] = int(out.group(1)) 
137              out = re.search(r'skips=(\d+)', l) 
138              if out: res['skips'] = int(out.group(1)) 
139              out = re.search(r'expectedFailures=(\d+)', l) 
140              if out: res['expectedFailures'] = int(out.group(1)) 
141              out = re.search(r'unexpectedSuccesses=(\d+)', l) 
142              if out: res['unexpectedSuccesses'] = int(out.group(1)) 
143              # successes= is a Twisted-2.0 addition, and is not currently used 
144              out = re.search(r'successes=(\d+)', l) 
145              if out: res['successes'] = int(out.group(1)) 
146   
147      return res 
148   
149   
150 -class TrialTestCaseCounter(LogLineObserver): 
151      _line_re = re.compile(r'^(?:Doctest: )?([\w\.]+) \.\.\. \[([^\]]+)\]$') 
152      numTests = 0 
153      finished = False 
154   
155 -    def outLineReceived(self, line): 
156          # different versions of Twisted emit different per-test lines with 
157          # the bwverbose reporter. 
158          #  2.0.0: testSlave (buildbot.test.test_runner.Create) ... [OK] 
159          #  2.1.0: buildbot.test.test_runner.Create.testSlave ... [OK] 
160          #  2.4.0: buildbot.test.test_runner.Create.testSlave ... [OK] 
161          # Let's just handle the most recent version, since it's the easiest. 
162          # Note that doctests create lines line this: 
163          #  Doctest: viff.field.GF ... [OK] 
164   
165          if self.finished: 
166              return 
167          if line.startswith("=" * 40): 
168              self.finished = True 
169              return 
170   
171          m = self._line_re.search(line.strip()) 
172          if m: 
173              testname, result = m.groups() 
174              self.numTests += 1 
175              self.step.setProgress('tests', self.numTests) 
176   
177   
178  UNSPECIFIED=() # since None is a valid choice 
179   
180 -class Trial(ShellCommand): 
181      """ 
182      There are some class attributes which may be usefully overridden 
183      by subclasses. 'trialMode' and 'trialArgs' can influence the trial 
184      command line. 
185      """ 
186   
187      name = "trial" 
188      progressMetrics = ('output', 'tests', 'test.log') 
189      # note: the slash only works on unix buildslaves, of course, but we have 
190      # no way to know what the buildslave uses as a separator.  
191      # TODO: figure out something clever. 
192      logfiles = {"test.log": "_trial_temp/test.log"} 
193      # we use test.log to track Progress at the end of __init__() 
194   
195      renderables = ['tests'] 
196      flunkOnFailure = True 
197      python = None 
198      trial = "trial" 
199      trialMode = ["--reporter=bwverbose"] # requires Twisted-2.1.0 or newer 
200      # for Twisted-2.0.0 or 1.3.0, use ["-o"] instead 
201      trialArgs = [] 
202      testpath = UNSPECIFIED # required (but can be None) 
203      testChanges = False # TODO: needs better name 
204      recurse = False 
205      reactor = None 
206      randomly = False 
207      tests = None # required 
208   
209 -    def __init__(self, reactor=UNSPECIFIED, python=None, trial=None, 
210                   testpath=UNSPECIFIED, 
211                   tests=None, testChanges=None, 
212                   recurse=None, randomly=None, 
213                   trialMode=None, trialArgs=None, 
214                   **kwargs): 
215          """ 
216          @type  testpath: string 
217          @param testpath: use in PYTHONPATH when running the tests. If 
218                           None, do not set PYTHONPATH. Setting this to '.' will 
219                           cause the source files to be used in-place. 
220   
221          @type  python: string (without spaces) or list 
222          @param python: which python executable to use. Will form the start of 
223                         the argv array that will launch trial. If you use this, 
224                         you should set 'trial' to an explicit path (like 
225                         /usr/bin/trial or ./bin/trial). Defaults to None, which 
226                         leaves it out entirely (running 'trial args' instead of 
227                         'python ./bin/trial args'). Likely values are 'python', 
228                         ['python2.2'], ['python', '-Wall'], etc. 
229   
230          @type  trial: string 
231          @param trial: which 'trial' executable to run. 
232                        Defaults to 'trial', which will cause $PATH to be 
233                        searched and probably find /usr/bin/trial . If you set 
234                        'python', this should be set to an explicit path (because 
235                        'python2.3 trial' will not work). 
236   
237          @type trialMode: list of strings 
238          @param trialMode: a list of arguments to pass to trial, specifically 
239                            to set the reporting mode. This defaults to ['-to'] 
240                            which means 'verbose colorless output' to the trial 
241                            that comes with Twisted-2.0.x and at least -2.1.0 . 
242                            Newer versions of Twisted may come with a trial 
243                            that prefers ['--reporter=bwverbose']. 
244   
245          @type trialArgs: list of strings 
246          @param trialArgs: a list of arguments to pass to trial, available to 
247                            turn on any extra flags you like. Defaults to []. 
248   
249          @type  tests: list of strings 
250          @param tests: a list of test modules to run, like 
251                        ['twisted.test.test_defer', 'twisted.test.test_process']. 
252                        If this is a string, it will be converted into a one-item 
253                        list. 
254   
255          @type  testChanges: boolean 
256          @param testChanges: if True, ignore the 'tests' parameter and instead 
257                              ask the Build for all the files that make up the 
258                              Changes going into this build. Pass these filenames 
259                              to trial and ask it to look for test-case-name 
260                              tags, running just the tests necessary to cover the 
261                              changes. 
262   
263          @type  recurse: boolean 
264          @param recurse: If True, pass the --recurse option to trial, allowing 
265                          test cases to be found in deeper subdirectories of the 
266                          modules listed in 'tests'. This does not appear to be 
267                          necessary when using testChanges. 
268   
269          @type  reactor: string 
270          @param reactor: which reactor to use, like 'gtk' or 'java'. If not 
271                          provided, the Twisted's usual platform-dependent 
272                          default is used. 
273   
274          @type  randomly: boolean 
275          @param randomly: if True, add the --random=0 argument, which instructs 
276                           trial to run the unit tests in a random order each 
277                           time. This occasionally catches problems that might be 
278                           masked when one module always runs before another 
279                           (like failing to make registerAdapter calls before 
280                           lookups are done). 
281   
282          @type  kwargs: dict 
283          @param kwargs: parameters. The following parameters are inherited from 
284                         L{ShellCommand} and may be useful to set: workdir, 
285                         haltOnFailure, flunkOnWarnings, flunkOnFailure, 
286                         warnOnWarnings, warnOnFailure, want_stdout, want_stderr, 
287                         timeout. 
288          """ 
289          ShellCommand.__init__(self, **kwargs) 
290          self.addFactoryArguments(reactor=reactor, 
291                                   python=python, 
292                                   trial=trial, 
293                                   testpath=testpath, 
294                                   tests=tests, 
295                                   testChanges=testChanges, 
296                                   recurse=recurse, 
297                                   randomly=randomly, 
298                                   trialMode=trialMode, 
299                                   trialArgs=trialArgs, 
300                                   ) 
301   
302          if python: 
303              self.python = python 
304          if self.python is not None: 
305              if type(self.python) is str: 
306                  self.python = [self.python] 
307              for s in self.python: 
308                  if " " in s: 
309                      # this is not strictly an error, but I suspect more 
310                      # people will accidentally try to use python="python2.3 
311                      # -Wall" than will use embedded spaces in a python flag 
312                      log.msg("python= component '%s' has spaces") 
313                      log.msg("To add -Wall, use python=['python', '-Wall']") 
314                      why = "python= value has spaces, probably an error" 
315                      raise ValueError(why) 
316   
317          if trial: 
318              self.trial = trial 
319          if " " in self.trial: 
320              raise ValueError("trial= value has spaces") 
321          if trialMode is not None: 
322              self.trialMode = trialMode 
323          if trialArgs is not None: 
324              self.trialArgs = trialArgs 
325   
326          if testpath is not UNSPECIFIED: 
327              self.testpath = testpath 
328          if self.testpath is UNSPECIFIED: 
329              raise ValueError("You must specify testpath= (it can be None)") 
330          assert isinstance(self.testpath, str) or self.testpath is None 
331   
332          if reactor is not UNSPECIFIED: 
333              self.reactor = reactor 
334   
335          if tests is not None: 
336              self.tests = tests 
337          if type(self.tests) is str: 
338              self.tests = [self.tests] 
339          if testChanges is not None: 
340              self.testChanges = testChanges 
341              #self.recurse = True  # not sure this is necessary 
342   
343          if not self.testChanges and self.tests is None: 
344              raise ValueError("Must either set testChanges= or provide tests=") 
345   
346          if recurse is not None: 
347              self.recurse = recurse 
348          if randomly is not None: 
349              self.randomly = randomly 
350   
351          # build up most of the command, then stash it until start() 
352          command = [] 
353          if self.python: 
354              command.extend(self.python) 
355          command.append(self.trial) 
356          command.extend(self.trialMode) 
357          if self.recurse: 
358              command.append("--recurse") 
359          if self.reactor: 
360              command.append("--reactor=%s" % reactor) 
361          if self.randomly: 
362              command.append("--random=0") 
363          command.extend(self.trialArgs) 
364          self.command = command 
365   
366          if self.reactor: 
367              self.description = ["testing", "(%s)" % self.reactor] 
368              self.descriptionDone = ["tests"] 
369              # commandComplete adds (reactorname) to self.text 
370          else: 
371              self.description = ["testing"] 
372              self.descriptionDone = ["tests"] 
373   
374          # this counter will feed Progress along the 'test cases' metric 
375          self.addLogObserver('stdio', TrialTestCaseCounter()) 
376          # this one just measures bytes of output in _trial_temp/test.log 
377          self.addLogObserver('test.log', OutputProgressObserver('test.log')) 
378   
379 -    def setupEnvironment(self, cmd): 
380          ShellCommand.setupEnvironment(self, cmd) 
381          if self.testpath != None: 
382              e = cmd.args['env'] 
383              if e is None: 
384                  cmd.args['env'] = {'PYTHONPATH': self.testpath} 
385              else: 
386                  #this bit produces a list, which can be used 
387                  #by buildslave.runprocess.RunProcess 
388                  ppath = e.get('PYTHONPATH', self.testpath) 
389                  if isinstance(ppath, str): 
390                      ppath = [ppath] 
391                  if self.testpath not in ppath: 
392                      ppath.insert(0, self.testpath) 
393                  e['PYTHONPATH'] = ppath 
394   
395 -    def start(self): 
396          # now that self.build.allFiles() is nailed down, finish building the 
397          # command 
398          if self.testChanges: 
399              for f in self.build.allFiles(): 
400                  if f.endswith(".py"): 
401                      self.command.append("--testmodule=%s" % f) 
402          else: 
403              self.command.extend(self.tests) 
404          log.msg("Trial.start: command is", self.command) 
405   
406          ShellCommand.start(self) 
407   
408   
409 -    def commandComplete(self, cmd): 
410          # figure out all status, then let the various hook functions return 
411          # different pieces of it 
412   
413          # 'cmd' is the original trial command, so cmd.logs['stdio'] is the 
414          # trial output. We don't have access to test.log from here. 
415          output = cmd.logs['stdio'].getText() 
416          counts = countFailedTests(output) 
417   
418          total = counts['total'] 
419          failures, errors = counts['failures'], counts['errors'] 
420          parsed = (total != None) 
421          text = [] 
422          text2 = "" 
423   
424          if cmd.rc == 0: 
425              if parsed: 
426                  results = SUCCESS 
427                  if total: 
428                      text += ["%d %s" % \ 
429                               (total, 
430                                total == 1 and "test" or "tests"), 
431                               "passed"] 
432                  else: 
433                      text += ["no tests", "run"] 
434              else: 
435                  results = FAILURE 
436                  text += ["testlog", "unparseable"] 
437                  text2 = "tests" 
438          else: 
439              # something failed 
440              results = FAILURE 
441              if parsed: 
442                  text.append("tests") 
443                  if failures: 
444                      text.append("%d %s" % \ 
445                                  (failures, 
446                                   failures == 1 and "failure" or "failures")) 
447                  if errors: 
448                      text.append("%d %s" % \ 
449                                  (errors, 
450                                   errors == 1 and "error" or "errors")) 
451                  count = failures + errors 
452                  text2 = "%d tes%s" % (count, (count == 1 and 't' or 'ts')) 
453              else: 
454                  text += ["tests", "failed"] 
455                  text2 = "tests" 
456   
457          if counts['skips']: 
458              text.append("%d %s" %  \ 
459                          (counts['skips'], 
460                           counts['skips'] == 1 and "skip" or "skips")) 
461          if counts['expectedFailures']: 
462              text.append("%d %s" %  \ 
463                          (counts['expectedFailures'], 
464                           counts['expectedFailures'] == 1 and "todo" 
465                           or "todos")) 
466              if 0: # TODO 
467                  results = WARNINGS 
468                  if not text2: 
469                      text2 = "todo" 
470   
471          if 0: 
472              # ignore unexpectedSuccesses for now, but it should really mark 
473              # the build WARNING 
474              if counts['unexpectedSuccesses']: 
475                  text.append("%d surprises" % counts['unexpectedSuccesses']) 
476                  results = WARNINGS 
477                  if not text2: 
478                      text2 = "tests" 
479   
480          if self.reactor: 
481              text.append(self.rtext('(%s)')) 
482              if text2: 
483                  text2 = "%s %s" % (text2, self.rtext('(%s)')) 
484   
485          self.results = results 
486          self.text = text 
487          self.text2 = [text2] 
488   
489   
490 -    def rtext(self, fmt='%s'): 
491          if self.reactor: 
492              rtext = fmt % self.reactor 
493              return rtext.replace("reactor", "") 
494          return "" 
495   
496 -    def addTestResult(self, testname, results, text, tlog): 
497          if self.reactor is not None: 
498              testname = (self.reactor,) + testname 
499          tr = testresult.TestResult(testname, results, text, logs={'log': tlog}) 
500          #self.step_status.build.addTestResult(tr) 
501          self.build.build_status.addTestResult(tr) 
502   
503 -    def createSummary(self, loog): 
504          output = loog.getText() 
505          problems = "" 
506          sio = StringIO.StringIO(output) 
507          warnings = {} 
508          while 1: 
509              line = sio.readline() 
510              if line == "": 
511                  break 
512              if line.find(" exceptions.DeprecationWarning: ") != -1: 
513                  # no source 
514                  warning = line # TODO: consider stripping basedir prefix here 
515                  warnings[warning] = warnings.get(warning, 0) + 1 
516              elif (line.find(" DeprecationWarning: ") != -1 or 
517                  line.find(" UserWarning: ") != -1): 
518                  # next line is the source 
519                  warning = line + sio.readline() 
520                  warnings[warning] = warnings.get(warning, 0) + 1 
521              elif line.find("Warning: ") != -1: 
522                  warning = line 
523                  warnings[warning] = warnings.get(warning, 0) + 1 
524   
525              if line.find("=" * 60) == 0 or line.find("-" * 60) == 0: 
526                  problems += line 
527                  problems += sio.read() 
528                  break 
529   
530          if problems: 
531              self.addCompleteLog("problems", problems) 
532              # now parse the problems for per-test results 
533              pio = StringIO.StringIO(problems) 
534              pio.readline() # eat the first separator line 
535              testname = None 
536              done = False 
537              while not done: 
538                  while 1: 
539                      line = pio.readline() 
540                      if line == "": 
541                          done = True 
542                          break 
543                      if line.find("=" * 60) == 0: 
544                          break 
545                      if line.find("-" * 60) == 0: 
546                          # the last case has --- as a separator before the 
547                          # summary counts are printed 
548                          done = True 
549                          break 
550                      if testname is None: 
551                          # the first line after the === is like: 
552  # EXPECTED FAILURE: testLackOfTB (twisted.test.test_failure.FailureTestCase) 
553  # SKIPPED: testRETR (twisted.test.test_ftp.TestFTPServer) 
554  # FAILURE: testBatchFile (twisted.conch.test.test_sftp.TestOurServerBatchFile) 
555                          r = re.search(r'^([^:]+): (\w+) \(([\w\.]+)\)', line) 
556                          if not r: 
557                              # TODO: cleanup, if there are no problems, 
558                              # we hit here 
559                              continue 
560                          result, name, case = r.groups() 
561                          testname = tuple(case.split(".") + [name]) 
562                          results = {'SKIPPED': SKIPPED, 
563                                     'EXPECTED FAILURE': SUCCESS, 
564                                     'UNEXPECTED SUCCESS': WARNINGS, 
565                                     'FAILURE': FAILURE, 
566                                     'ERROR': FAILURE, 
567                                     'SUCCESS': SUCCESS, # not reported 
568                                     }.get(result, WARNINGS) 
569                          text = result.lower().split() 
570                          loog = line 
571                          # the next line is all dashes 
572                          loog += pio.readline() 
573                      else: 
574                          # the rest goes into the log 
575                          loog += line 
576                  if testname: 
577                      self.addTestResult(testname, results, text, loog) 
578                      testname = None 
579   
580          if warnings: 
581              lines = warnings.keys() 
582              lines.sort() 
583              self.addCompleteLog("warnings", "".join(lines)) 
584   
585 -    def evaluateCommand(self, cmd): 
586          return self.results 
587   
588 -    def getText(self, cmd, results): 
589          return self.text 
590 -    def getText2(self, cmd, results): 
591          return self.text2 
592   
593   
594 -class RemovePYCs(ShellCommand): 
595      name = "remove-.pyc" 
596      command = ['find', '.', '-name', '*.pyc', '-exec', 'rm', '{}', ';'] 
597      description = ["removing", ".pyc", "files"] 
598      descriptionDone = ["remove", ".pycs"] 
599