Source Code for Module buildbot.steps.python_twisted

  1  # This file is part of Buildbot.  Buildbot is free software: you can 
  2  # redistribute it and/or modify it under the terms of the GNU General Public 
  3  # License as published by the Free Software Foundation, version 2. 
  4  # 
  5  # This program is distributed in the hope that it will be useful, but WITHOUT 
  6  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
  7  # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more 
  8  # details. 
  9  # 
 10  # You should have received a copy of the GNU General Public License along with 
 11  # this program; if not, write to the Free Software Foundation, Inc., 51 
 12  # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 
 13  # 
 14  # Copyright Buildbot Team Members 
 15   
 16   
 17  from twisted.python import log 
 18   
 19  from buildbot.status import testresult 
 20  from buildbot.status.results import SUCCESS, FAILURE, WARNINGS, SKIPPED 
 21  from buildbot.process.buildstep import LogLineObserver, OutputProgressObserver 
 22  from buildbot.steps.shell import ShellCommand 
 23   
 24  try: 
 25      import cStringIO 
 26      StringIO = cStringIO 
 27  except ImportError: 
 28      import StringIO 
 29  import re 
 30   
 31  # BuildSteps that are specific to the Twisted source tree 
 32   
 33 -class HLint(ShellCommand): 
 34      """I run a 'lint' checker over a set of .xhtml files. Any deviations 
 35      from recommended style is flagged and put in the output log. 
 36   
 37      This step looks at .changes in the parent Build to extract a list of 
 38      Lore XHTML files to check.""" 
 39   
 40      name = "hlint" 
 41      description = ["running", "hlint"] 
 42      descriptionDone = ["hlint"] 
 43      warnOnWarnings = True 
 44      warnOnFailure = True 
 45      # TODO: track time, but not output 
 46      warnings = 0 
 47   
 48 -    def __init__(self, python=None, **kwargs): 
 49          ShellCommand.__init__(self, **kwargs) 
 50          self.addFactoryArguments(python=python) 
 51          self.python = python 
 52   
 53 -    def start(self): 
 54          # create the command 
 55          htmlFiles = {} 
 56          for f in self.build.allFiles(): 
 57              if f.endswith(".xhtml") and not f.startswith("sandbox/"): 
 58                  htmlFiles[f] = 1 
 59          # remove duplicates 
 60          hlintTargets = htmlFiles.keys() 
 61          hlintTargets.sort() 
 62          if not hlintTargets: 
 63              return SKIPPED 
 64          self.hlintFiles = hlintTargets 
 65          c = [] 
 66          if self.python: 
 67              c.append(self.python) 
 68          c += ["bin/lore", "-p", "--output", "lint"] + self.hlintFiles 
 69          self.setCommand(c) 
 70   
 71          # add an extra log file to show the .html files we're checking 
 72          self.addCompleteLog("files", "\n".join(self.hlintFiles)+"\n") 
 73   
 74          ShellCommand.start(self) 
 75   
 76 -    def commandComplete(self, cmd): 
 77          # TODO: remove the 'files' file (a list of .xhtml files that were 
 78          # submitted to hlint) because it is available in the logfile and 
 79          # mostly exists to give the user an idea of how long the step will 
 80          # take anyway). 
 81          lines = cmd.logs['stdio'].getText().split("\n") 
 82          warningLines = filter(lambda line:':' in line, lines) 
 83          if warningLines: 
 84              self.addCompleteLog("warnings", "".join(warningLines)) 
 85          warnings = len(warningLines) 
 86          self.warnings = warnings 
 87   
 88 -    def evaluateCommand(self, cmd): 
 89          # warnings are in stdout, rc is always 0, unless the tools break 
 90          if cmd.rc != 0: 
 91              return FAILURE 
 92          if self.warnings: 
 93              return WARNINGS 
 94          return SUCCESS 
 95   
 96 -    def getText2(self, cmd, results): 
 97          if cmd.rc != 0: 
 98              return ["hlint"] 
 99          return ["%d hlin%s" % (self.warnings, 
100                                 self.warnings == 1 and 't' or 'ts')] 
101   
102 -def countFailedTests(output): 
103      # start scanning 10kb from the end, because there might be a few kb of 
104      # import exception tracebacks between the total/time line and the errors 
105      # line 
106      chunk = output[-10000:] 
107      lines = chunk.split("\n") 
108      lines.pop() # blank line at end 
109      # lines[-3] is "Ran NN tests in 0.242s" 
110      # lines[-2] is blank 
111      # lines[-1] is 'OK' or 'FAILED (failures=1, errors=12)' 
112      #  or 'FAILED (failures=1)' 
113      #  or "PASSED (skips=N, successes=N)"  (for Twisted-2.0) 
114      # there might be other lines dumped here. Scan all the lines. 
115      res = {'total': None, 
116             'failures': 0, 
117             'errors': 0, 
118             'skips': 0, 
119             'expectedFailures': 0, 
120             'unexpectedSuccesses': 0, 
121             } 
122      for l in lines: 
123          out = re.search(r'Ran (\d+) tests', l) 
124          if out: 
125              res['total'] = int(out.group(1)) 
126          if (l.startswith("OK") or 
127              l.startswith("FAILED ") or 
128              l.startswith("PASSED")): 
129              # the extra space on FAILED_ is to distinguish the overall 
130              # status from an individual test which failed. The lack of a 
131              # space on the OK is because it may be printed without any 
132              # additional text (if there are no skips,etc) 
133              out = re.search(r'failures=(\d+)', l) 
134              if out: res['failures'] = int(out.group(1)) 
135              out = re.search(r'errors=(\d+)', l) 
136              if out: res['errors'] = int(out.group(1)) 
137              out = re.search(r'skips=(\d+)', l) 
138              if out: res['skips'] = int(out.group(1)) 
139              out = re.search(r'expectedFailures=(\d+)', l) 
140              if out: res['expectedFailures'] = int(out.group(1)) 
141              out = re.search(r'unexpectedSuccesses=(\d+)', l) 
142              if out: res['unexpectedSuccesses'] = int(out.group(1)) 
143              # successes= is a Twisted-2.0 addition, and is not currently used 
144              out = re.search(r'successes=(\d+)', l) 
145              if out: res['successes'] = int(out.group(1)) 
146   
147      return res 
148   
149   
150 -class TrialTestCaseCounter(LogLineObserver): 
151      _line_re = re.compile(r'^(?:Doctest: )?([\w\.]+) \.\.\. \[([^\]]+)\]$') 
152      numTests = 0 
153      finished = False 
154   
155 -    def outLineReceived(self, line): 
156          # different versions of Twisted emit different per-test lines with 
157          # the bwverbose reporter. 
158          #  2.0.0: testSlave (buildbot.test.test_runner.Create) ... [OK] 
159          #  2.1.0: buildbot.test.test_runner.Create.testSlave ... [OK] 
160          #  2.4.0: buildbot.test.test_runner.Create.testSlave ... [OK] 
161          # Let's just handle the most recent version, since it's the easiest. 
162          # Note that doctests create lines line this: 
163          #  Doctest: viff.field.GF ... [OK] 
164   
165          if self.finished: 
166              return 
167          if line.startswith("=" * 40): 
168              self.finished = True 
169              return 
170   
171          m = self._line_re.search(line.strip()) 
172          if m: 
173              testname, result = m.groups() 
174              self.numTests += 1 
175              self.step.setProgress('tests', self.numTests) 
176   
177   
178  UNSPECIFIED=() # since None is a valid choice 
179   
180 -class Trial(ShellCommand): 
181      """ 
182      There are some class attributes which may be usefully overridden 
183      by subclasses. 'trialMode' and 'trialArgs' can influence the trial 
184      command line. 
185      """ 
186   
187      name = "trial" 
188      progressMetrics = ('output', 'tests', 'test.log') 
189      # note: the slash only works on unix buildslaves, of course, but we have 
190      # no way to know what the buildslave uses as a separator.  
191      # TODO: figure out something clever. 
192      logfiles = {"test.log": "_trial_temp/test.log"} 
193      # we use test.log to track Progress at the end of __init__() 
194   
195      flunkOnFailure = True 
196      python = None 
197      trial = "trial" 
198      trialMode = ["--reporter=bwverbose"] # requires Twisted-2.1.0 or newer 
199      # for Twisted-2.0.0 or 1.3.0, use ["-o"] instead 
200      trialArgs = [] 
201      testpath = UNSPECIFIED # required (but can be None) 
202      testChanges = False # TODO: needs better name 
203      recurse = False 
204      reactor = None 
205      randomly = False 
206      tests = None # required 
207   
208 -    def __init__(self, reactor=UNSPECIFIED, python=None, trial=None, 
209                   testpath=UNSPECIFIED, 
210                   tests=None, testChanges=None, 
211                   recurse=None, randomly=None, 
212                   trialMode=None, trialArgs=None, 
213                   **kwargs): 
214          """ 
215          @type  testpath: string 
216          @param testpath: use in PYTHONPATH when running the tests. If 
217                           None, do not set PYTHONPATH. Setting this to '.' will 
218                           cause the source files to be used in-place. 
219   
220          @type  python: string (without spaces) or list 
221          @param python: which python executable to use. Will form the start of 
222                         the argv array that will launch trial. If you use this, 
223                         you should set 'trial' to an explicit path (like 
224                         /usr/bin/trial or ./bin/trial). Defaults to None, which 
225                         leaves it out entirely (running 'trial args' instead of 
226                         'python ./bin/trial args'). Likely values are 'python', 
227                         ['python2.2'], ['python', '-Wall'], etc. 
228   
229          @type  trial: string 
230          @param trial: which 'trial' executable to run. 
231                        Defaults to 'trial', which will cause $PATH to be 
232                        searched and probably find /usr/bin/trial . If you set 
233                        'python', this should be set to an explicit path (because 
234                        'python2.3 trial' will not work). 
235   
236          @type trialMode: list of strings 
237          @param trialMode: a list of arguments to pass to trial, specifically 
238                            to set the reporting mode. This defaults to ['-to'] 
239                            which means 'verbose colorless output' to the trial 
240                            that comes with Twisted-2.0.x and at least -2.1.0 . 
241                            Newer versions of Twisted may come with a trial 
242                            that prefers ['--reporter=bwverbose']. 
243   
244          @type trialArgs: list of strings 
245          @param trialArgs: a list of arguments to pass to trial, available to 
246                            turn on any extra flags you like. Defaults to []. 
247   
248          @type  tests: list of strings 
249          @param tests: a list of test modules to run, like 
250                        ['twisted.test.test_defer', 'twisted.test.test_process']. 
251                        If this is a string, it will be converted into a one-item 
252                        list. 
253   
254          @type  testChanges: boolean 
255          @param testChanges: if True, ignore the 'tests' parameter and instead 
256                              ask the Build for all the files that make up the 
257                              Changes going into this build. Pass these filenames 
258                              to trial and ask it to look for test-case-name 
259                              tags, running just the tests necessary to cover the 
260                              changes. 
261   
262          @type  recurse: boolean 
263          @param recurse: If True, pass the --recurse option to trial, allowing 
264                          test cases to be found in deeper subdirectories of the 
265                          modules listed in 'tests'. This does not appear to be 
266                          necessary when using testChanges. 
267   
268          @type  reactor: string 
269          @param reactor: which reactor to use, like 'gtk' or 'java'. If not 
270                          provided, the Twisted's usual platform-dependent 
271                          default is used. 
272   
273          @type  randomly: boolean 
274          @param randomly: if True, add the --random=0 argument, which instructs 
275                           trial to run the unit tests in a random order each 
276                           time. This occasionally catches problems that might be 
277                           masked when one module always runs before another 
278                           (like failing to make registerAdapter calls before 
279                           lookups are done). 
280   
281          @type  kwargs: dict 
282          @param kwargs: parameters. The following parameters are inherited from 
283                         L{ShellCommand} and may be useful to set: workdir, 
284                         haltOnFailure, flunkOnWarnings, flunkOnFailure, 
285                         warnOnWarnings, warnOnFailure, want_stdout, want_stderr, 
286                         timeout. 
287          """ 
288          ShellCommand.__init__(self, **kwargs) 
289          self.addFactoryArguments(reactor=reactor, 
290                                   python=python, 
291                                   trial=trial, 
292                                   testpath=testpath, 
293                                   tests=tests, 
294                                   testChanges=testChanges, 
295                                   recurse=recurse, 
296                                   randomly=randomly, 
297                                   trialMode=trialMode, 
298                                   trialArgs=trialArgs, 
299                                   ) 
300   
301          if python: 
302              self.python = python 
303          if self.python is not None: 
304              if type(self.python) is str: 
305                  self.python = [self.python] 
306              for s in self.python: 
307                  if " " in s: 
308                      # this is not strictly an error, but I suspect more 
309                      # people will accidentally try to use python="python2.3 
310                      # -Wall" than will use embedded spaces in a python flag 
311                      log.msg("python= component '%s' has spaces") 
312                      log.msg("To add -Wall, use python=['python', '-Wall']") 
313                      why = "python= value has spaces, probably an error" 
314                      raise ValueError(why) 
315   
316          if trial: 
317              self.trial = trial 
318          if " " in self.trial: 
319              raise ValueError("trial= value has spaces") 
320          if trialMode is not None: 
321              self.trialMode = trialMode 
322          if trialArgs is not None: 
323              self.trialArgs = trialArgs 
324   
325          if testpath is not UNSPECIFIED: 
326              self.testpath = testpath 
327          if self.testpath is UNSPECIFIED: 
328              raise ValueError("You must specify testpath= (it can be None)") 
329          assert isinstance(self.testpath, str) or self.testpath is None 
330   
331          if reactor is not UNSPECIFIED: 
332              self.reactor = reactor 
333   
334          if tests is not None: 
335              self.tests = tests 
336          if type(self.tests) is str: 
337              self.tests = [self.tests] 
338          if testChanges is not None: 
339              self.testChanges = testChanges 
340              #self.recurse = True  # not sure this is necessary 
341   
342          if not self.testChanges and self.tests is None: 
343              raise ValueError("Must either set testChanges= or provide tests=") 
344   
345          if recurse is not None: 
346              self.recurse = recurse 
347          if randomly is not None: 
348              self.randomly = randomly 
349   
350          # build up most of the command, then stash it until start() 
351          command = [] 
352          if self.python: 
353              command.extend(self.python) 
354          command.append(self.trial) 
355          command.extend(self.trialMode) 
356          if self.recurse: 
357              command.append("--recurse") 
358          if self.reactor: 
359              command.append("--reactor=%s" % reactor) 
360          if self.randomly: 
361              command.append("--random=0") 
362          command.extend(self.trialArgs) 
363          self.command = command 
364   
365          if self.reactor: 
366              self.description = ["testing", "(%s)" % self.reactor] 
367              self.descriptionDone = ["tests"] 
368              # commandComplete adds (reactorname) to self.text 
369          else: 
370              self.description = ["testing"] 
371              self.descriptionDone = ["tests"] 
372   
373          # this counter will feed Progress along the 'test cases' metric 
374          self.addLogObserver('stdio', TrialTestCaseCounter()) 
375          # this one just measures bytes of output in _trial_temp/test.log 
376          self.addLogObserver('test.log', OutputProgressObserver('test.log')) 
377   
378 -    def setupEnvironment(self, cmd): 
379          ShellCommand.setupEnvironment(self, cmd) 
380          if self.testpath != None: 
381              e = cmd.args['env'] 
382              if e is None: 
383                  cmd.args['env'] = {'PYTHONPATH': self.testpath} 
384              else: 
385                  # TODO: somehow, each build causes another copy of 
386                  # self.testpath to get prepended 
387                  if e.get('PYTHONPATH', "") == "": 
388                      e['PYTHONPATH'] = self.testpath 
389                  else: 
390                      e['PYTHONPATH'] = self.testpath + ":" + e['PYTHONPATH'] 
391          try: 
392              p = cmd.args['env']['PYTHONPATH'] 
393              if type(p) is not str: 
394                  log.msg("hey, not a string:", p) 
395                  assert False 
396          except (KeyError, TypeError): 
397              # KeyError if args doesn't have ['env'] 
398              # KeyError if args['env'] doesn't have ['PYTHONPATH'] 
399              # TypeError if args is None 
400              pass 
401   
402 -    def start(self): 
403          # now that self.build.allFiles() is nailed down, finish building the 
404          # command 
405          if self.testChanges: 
406              for f in self.build.allFiles(): 
407                  if f.endswith(".py"): 
408                      self.command.append("--testmodule=%s" % f) 
409          else: 
410              self.command.extend(self.tests) 
411          log.msg("Trial.start: command is", self.command) 
412   
413          ShellCommand.start(self) 
414   
415   
416 -    def commandComplete(self, cmd): 
417          # figure out all status, then let the various hook functions return 
418          # different pieces of it 
419   
420          # 'cmd' is the original trial command, so cmd.logs['stdio'] is the 
421          # trial output. We don't have access to test.log from here. 
422          output = cmd.logs['stdio'].getText() 
423          counts = countFailedTests(output) 
424   
425          total = counts['total'] 
426          failures, errors = counts['failures'], counts['errors'] 
427          parsed = (total != None) 
428          text = [] 
429          text2 = "" 
430   
431          if cmd.rc == 0: 
432              if parsed: 
433                  results = SUCCESS 
434                  if total: 
435                      text += ["%d %s" % \ 
436                               (total, 
437                                total == 1 and "test" or "tests"), 
438                               "passed"] 
439                  else: 
440                      text += ["no tests", "run"] 
441              else: 
442                  results = FAILURE 
443                  text += ["testlog", "unparseable"] 
444                  text2 = "tests" 
445          else: 
446              # something failed 
447              results = FAILURE 
448              if parsed: 
449                  text.append("tests") 
450                  if failures: 
451                      text.append("%d %s" % \ 
452                                  (failures, 
453                                   failures == 1 and "failure" or "failures")) 
454                  if errors: 
455                      text.append("%d %s" % \ 
456                                  (errors, 
457                                   errors == 1 and "error" or "errors")) 
458                  count = failures + errors 
459                  text2 = "%d tes%s" % (count, (count == 1 and 't' or 'ts')) 
460              else: 
461                  text += ["tests", "failed"] 
462                  text2 = "tests" 
463   
464          if counts['skips']: 
465              text.append("%d %s" %  \ 
466                          (counts['skips'], 
467                           counts['skips'] == 1 and "skip" or "skips")) 
468          if counts['expectedFailures']: 
469              text.append("%d %s" %  \ 
470                          (counts['expectedFailures'], 
471                           counts['expectedFailures'] == 1 and "todo" 
472                           or "todos")) 
473              if 0: # TODO 
474                  results = WARNINGS 
475                  if not text2: 
476                      text2 = "todo" 
477   
478          if 0: 
479              # ignore unexpectedSuccesses for now, but it should really mark 
480              # the build WARNING 
481              if counts['unexpectedSuccesses']: 
482                  text.append("%d surprises" % counts['unexpectedSuccesses']) 
483                  results = WARNINGS 
484                  if not text2: 
485                      text2 = "tests" 
486   
487          if self.reactor: 
488              text.append(self.rtext('(%s)')) 
489              if text2: 
490                  text2 = "%s %s" % (text2, self.rtext('(%s)')) 
491   
492          self.results = results 
493          self.text = text 
494          self.text2 = [text2] 
495   
496   
497 -    def rtext(self, fmt='%s'): 
498          if self.reactor: 
499              rtext = fmt % self.reactor 
500              return rtext.replace("reactor", "") 
501          return "" 
502   
503 -    def addTestResult(self, testname, results, text, tlog): 
504          if self.reactor is not None: 
505              testname = (self.reactor,) + testname 
506          tr = testresult.TestResult(testname, results, text, logs={'log': tlog}) 
507          #self.step_status.build.addTestResult(tr) 
508          self.build.build_status.addTestResult(tr) 
509   
510 -    def createSummary(self, loog): 
511          output = loog.getText() 
512          problems = "" 
513          sio = StringIO.StringIO(output) 
514          warnings = {} 
515          while 1: 
516              line = sio.readline() 
517              if line == "": 
518                  break 
519              if line.find(" exceptions.DeprecationWarning: ") != -1: 
520                  # no source 
521                  warning = line # TODO: consider stripping basedir prefix here 
522                  warnings[warning] = warnings.get(warning, 0) + 1 
523              elif (line.find(" DeprecationWarning: ") != -1 or 
524                  line.find(" UserWarning: ") != -1): 
525                  # next line is the source 
526                  warning = line + sio.readline() 
527                  warnings[warning] = warnings.get(warning, 0) + 1 
528              elif line.find("Warning: ") != -1: 
529                  warning = line 
530                  warnings[warning] = warnings.get(warning, 0) + 1 
531   
532              if line.find("=" * 60) == 0 or line.find("-" * 60) == 0: 
533                  problems += line 
534                  problems += sio.read() 
535                  break 
536   
537          if problems: 
538              self.addCompleteLog("problems", problems) 
539              # now parse the problems for per-test results 
540              pio = StringIO.StringIO(problems) 
541              pio.readline() # eat the first separator line 
542              testname = None 
543              done = False 
544              while not done: 
545                  while 1: 
546                      line = pio.readline() 
547                      if line == "": 
548                          done = True 
549                          break 
550                      if line.find("=" * 60) == 0: 
551                          break 
552                      if line.find("-" * 60) == 0: 
553                          # the last case has --- as a separator before the 
554                          # summary counts are printed 
555                          done = True 
556                          break 
557                      if testname is None: 
558                          # the first line after the === is like: 
559  # EXPECTED FAILURE: testLackOfTB (twisted.test.test_failure.FailureTestCase) 
560  # SKIPPED: testRETR (twisted.test.test_ftp.TestFTPServer) 
561  # FAILURE: testBatchFile (twisted.conch.test.test_sftp.TestOurServerBatchFile) 
562                          r = re.search(r'^([^:]+): (\w+) \(([\w\.]+)\)', line) 
563                          if not r: 
564                              # TODO: cleanup, if there are no problems, 
565                              # we hit here 
566                              continue 
567                          result, name, case = r.groups() 
568                          testname = tuple(case.split(".") + [name]) 
569                          results = {'SKIPPED': SKIPPED, 
570                                     'EXPECTED FAILURE': SUCCESS, 
571                                     'UNEXPECTED SUCCESS': WARNINGS, 
572                                     'FAILURE': FAILURE, 
573                                     'ERROR': FAILURE, 
574                                     'SUCCESS': SUCCESS, # not reported 
575                                     }.get(result, WARNINGS) 
576                          text = result.lower().split() 
577                          loog = line 
578                          # the next line is all dashes 
579                          loog += pio.readline() 
580                      else: 
581                          # the rest goes into the log 
582                          loog += line 
583                  if testname: 
584                      self.addTestResult(testname, results, text, loog) 
585                      testname = None 
586   
587          if warnings: 
588              lines = warnings.keys() 
589              lines.sort() 
590              self.addCompleteLog("warnings", "".join(lines)) 
591   
592 -    def evaluateCommand(self, cmd): 
593          return self.results 
594   
595 -    def getText(self, cmd, results): 
596          return self.text 
597 -    def getText2(self, cmd, results): 
598          return self.text2 
599   
600   
601 -class RemovePYCs(ShellCommand): 
602      name = "remove-.pyc" 
603      command = ['find', '.', '-name', '*.pyc', '-exec', 'rm', '{}', ';'] 
604      description = ["removing", ".pyc", "files"] 
605      descriptionDone = ["remove", ".pycs"] 
606