buildbot.steps.python

168 """I run a unit test suite using 'trial', a unittest-like testing 169 framework that comes with Twisted. Trial is used to implement Twisted's 170 own unit tests, and is the unittest-framework of choice for many projects 171 that use Twisted internally. 172 173 Projects that use trial typically have all their test cases in a 'test' 174 subdirectory of their top-level library directory. I.e. for my package 175 'petmail', the tests are in 'petmail/test/test_*.py'. More complicated 176 packages (like Twisted itself) may have multiple test directories, like 177 'twisted/test/test_*.py' for the core functionality and 178 'twisted/mail/test/test_*.py' for the email-specific tests. 179 180 To run trial tests, you run the 'trial' executable and tell it where the 181 test cases are located. The most common way of doing this is with a 182 module name. For petmail, I would run 'trial petmail.test' and it would 183 locate all the test_*.py files under petmail/test/, running every test 184 case it could find in them. Unlike the unittest.py that comes with 185 Python, you do not run the test_foo.py as a script; you always let trial 186 do the importing and running. The 'tests' parameter controls which tests 187 trial will run: it can be a string or a list of strings. 188 189 To find these test cases, you must set a PYTHONPATH that allows something 190 like 'import petmail.test' to work. For packages that don't use a 191 separate top-level 'lib' directory, PYTHONPATH=. will work, and will use 192 the test cases (and the code they are testing) in-place. 193 PYTHONPATH=build/lib or PYTHONPATH=build/lib.$ARCH are also useful when 194 you do a'setup.py build' step first. The 'testpath' attribute of this 195 class controls what PYTHONPATH= is set to. 196 197 Trial has the ability (through the --testmodule flag) to run only the set 198 of test cases named by special 'test-case-name' tags in source files. We 199 can get the list of changed source files from our parent Build and 200 provide them to trial, thus running the minimal set of test cases needed 201 to cover the Changes. This is useful for quick builds, especially in 202 trees with a lot of test cases. The 'testChanges' parameter controls this 203 feature: if set, it will override 'tests'. 204 205 The trial executable itself is typically just 'trial' (which is usually 206 found on your $PATH as /usr/bin/trial), but it can be overridden with the 207 'trial' parameter. This is useful for Twisted's own unittests, which want 208 to use the copy of bin/trial that comes with the sources. (when bin/trial 209 discovers that it is living in a subdirectory named 'Twisted', it assumes 210 it is being run from the source tree and adds that parent directory to 211 PYTHONPATH. Therefore the canonical way to run Twisted's own unittest 212 suite is './bin/trial twisted.test' rather than 'PYTHONPATH=. 213 /usr/bin/trial twisted.test', especially handy when /usr/bin/trial has 214 not yet been installed). 215 216 To influence the version of python being used for the tests, or to add 217 flags to the command, set the 'python' parameter. This can be a string 218 (like 'python2.2') or a list (like ['python2.3', '-Wall']). 219 220 Trial creates and switches into a directory named _trial_temp/ before 221 running the tests, and sends the twisted log (which includes all 222 exceptions) to a file named test.log . This file will be pulled up to 223 the master where it can be seen as part of the status output. 224 225 There are some class attributes which may be usefully overridden 226 by subclasses. 'trialMode' and 'trialArgs' can influence the trial 227 command line. 228 """ 229 230 name = "trial" 231 progressMetrics = ('output', 'tests', 'test.log') 232 # note: the slash only works on unix buildslaves, of course, but we have 233 # no way to know what the buildslave uses as a separator. TODO: figure 234 # out something clever. 235 logfiles = {"test.log": "_trial_temp/test.log"} 236 # we use test.log to track Progress at the end of __init__() 237 238 flunkOnFailure = True 239 python = None 240 trial = "trial" 241 trialMode = ["--reporter=bwverbose"] # requires Twisted-2.1.0 or newer 242 # for Twisted-2.0.0 or 1.3.0, use ["-o"] instead 243 trialArgs = [] 244 testpath = UNSPECIFIED # required (but can be None) 245 testChanges = False # TODO: needs better name 246 recurse = False 247 reactor = None 248 randomly = False 249 tests = None # required 250

251 - def __init__(self, reactor=UNSPECIFIED, python=None, trial=None, 252 testpath=UNSPECIFIED, 253 tests=None, testChanges=None, 254 recurse=None, randomly=None, 255 trialMode=None, trialArgs=None, 256 **kwargs):

257 """ 258 @type testpath: string 259 @param testpath: use in PYTHONPATH when running the tests. If 260 None, do not set PYTHONPATH. Setting this to '.' will 261 cause the source files to be used in-place. 262 263 @type python: string (without spaces) or list 264 @param python: which python executable to use. Will form the start of 265 the argv array that will launch trial. If you use this, 266 you should set 'trial' to an explicit path (like 267 /usr/bin/trial or ./bin/trial). Defaults to None, which 268 leaves it out entirely (running 'trial args' instead of 269 'python ./bin/trial args'). Likely values are 'python', 270 ['python2.2'], ['python', '-Wall'], etc. 271 272 @type trial: string 273 @param trial: which 'trial' executable to run. 274 Defaults to 'trial', which will cause $PATH to be 275 searched and probably find /usr/bin/trial . If you set 276 'python', this should be set to an explicit path (because 277 'python2.3 trial' will not work). 278 279 @type trialMode: list of strings 280 @param trialMode: a list of arguments to pass to trial, specifically 281 to set the reporting mode. This defaults to ['-to'] 282 which means 'verbose colorless output' to the trial 283 that comes with Twisted-2.0.x and at least -2.1.0 . 284 Newer versions of Twisted may come with a trial 285 that prefers ['--reporter=bwverbose']. 286 287 @type trialArgs: list of strings 288 @param trialArgs: a list of arguments to pass to trial, available to 289 turn on any extra flags you like. Defaults to []. 290 291 @type tests: list of strings 292 @param tests: a list of test modules to run, like 293 ['twisted.test.test_defer', 'twisted.test.test_process']. 294 If this is a string, it will be converted into a one-item 295 list. 296 297 @type testChanges: boolean 298 @param testChanges: if True, ignore the 'tests' parameter and instead 299 ask the Build for all the files that make up the 300 Changes going into this build. Pass these filenames 301 to trial and ask it to look for test-case-name 302 tags, running just the tests necessary to cover the 303 changes. 304 305 @type recurse: boolean 306 @param recurse: If True, pass the --recurse option to trial, allowing 307 test cases to be found in deeper subdirectories of the 308 modules listed in 'tests'. This does not appear to be 309 necessary when using testChanges. 310 311 @type reactor: string 312 @param reactor: which reactor to use, like 'gtk' or 'java'. If not 313 provided, the Twisted's usual platform-dependent 314 default is used. 315 316 @type randomly: boolean 317 @param randomly: if True, add the --random=0 argument, which instructs 318 trial to run the unit tests in a random order each 319 time. This occasionally catches problems that might be 320 masked when one module always runs before another 321 (like failing to make registerAdapter calls before 322 lookups are done). 323 324 @type kwargs: dict 325 @param kwargs: parameters. The following parameters are inherited from 326 L{ShellCommand} and may be useful to set: workdir, 327 haltOnFailure, flunkOnWarnings, flunkOnFailure, 328 warnOnWarnings, warnOnFailure, want_stdout, want_stderr, 329 timeout. 330 """ 331 ShellCommand.__init__(self, **kwargs) 332 self.addFactoryArguments(reactor=reactor, 333 python=python, 334 trial=trial, 335 testpath=testpath, 336 tests=tests, 337 testChanges=testChanges, 338 recurse=recurse, 339 randomly=randomly, 340 trialMode=trialMode, 341 trialArgs=trialArgs, 342 ) 343 344 if python: 345 self.python = python 346 if self.python is not None: 347 if type(self.python) is str: 348 self.python = [self.python] 349 for s in self.python: 350 if " " in s: 351 # this is not strictly an error, but I suspect more 352 # people will accidentally try to use python="python2.3 353 # -Wall" than will use embedded spaces in a python flag 354 log.msg("python= component '%s' has spaces") 355 log.msg("To add -Wall, use python=['python', '-Wall']") 356 why = "python= value has spaces, probably an error" 357 raise ValueError(why) 358 359 if trial: 360 self.trial = trial 361 if " " in self.trial: 362 raise ValueError("trial= value has spaces") 363 if trialMode is not None: 364 self.trialMode = trialMode 365 if trialArgs is not None: 366 self.trialArgs = trialArgs 367 368 if testpath is not UNSPECIFIED: 369 self.testpath = testpath 370 if self.testpath is UNSPECIFIED: 371 raise ValueError("You must specify testpath= (it can be None)") 372 assert isinstance(self.testpath, str) or self.testpath is None 373 374 if reactor is not UNSPECIFIED: 375 self.reactor = reactor 376 377 if tests is not None: 378 self.tests = tests 379 if type(self.tests) is str: 380 self.tests = [self.tests] 381 if testChanges is not None: 382 self.testChanges = testChanges 383 #self.recurse = True # not sure this is necessary 384 385 if not self.testChanges and self.tests is None: 386 raise ValueError("Must either set testChanges= or provide tests=") 387 388 if recurse is not None: 389 self.recurse = recurse 390 if randomly is not None: 391 self.randomly = randomly 392 393 # build up most of the command, then stash it until start() 394 command = [] 395 if self.python: 396 command.extend(self.python) 397 command.append(self.trial) 398 command.extend(self.trialMode) 399 if self.recurse: 400 command.append("--recurse") 401 if self.reactor: 402 command.append("--reactor=%s" % reactor) 403 if self.randomly: 404 command.append("--random=0") 405 command.extend(self.trialArgs) 406 self.command = command 407 408 if self.reactor: 409 self.description = ["testing", "(%s)" % self.reactor] 410 self.descriptionDone = ["tests"] 411 # commandComplete adds (reactorname) to self.text 412 else: 413 self.description = ["testing"] 414 self.descriptionDone = ["tests"] 415 416 # this counter will feed Progress along the 'test cases' metric 417 self.addLogObserver('stdio', TrialTestCaseCounter()) 418 # this one just measures bytes of output in _trial_temp/test.log 419 self.addLogObserver('test.log', OutputProgressObserver('test.log'))

420

421 - def setupEnvironment(self, cmd):

422 ShellCommand.setupEnvironment(self, cmd) 423 if self.testpath != None: 424 e = cmd.args['env'] 425 if e is None: 426 cmd.args['env'] = {'PYTHONPATH': self.testpath} 427 else: 428 # TODO: somehow, each build causes another copy of 429 # self.testpath to get prepended 430 if e.get('PYTHONPATH', "") == "": 431 e['PYTHONPATH'] = self.testpath 432 else: 433 e['PYTHONPATH'] = self.testpath + ":" + e['PYTHONPATH'] 434 try: 435 p = cmd.args['env']['PYTHONPATH'] 436 if type(p) is not str: 437 log.msg("hey, not a string:", p) 438 assert False 439 except (KeyError, TypeError): 440 # KeyError if args doesn't have ['env'] 441 # KeyError if args['env'] doesn't have ['PYTHONPATH'] 442 # TypeError if args is None 443 pass

444

445 - def start(self):

446 # now that self.build.allFiles() is nailed down, finish building the 447 # command 448 if self.testChanges: 449 for f in self.build.allFiles(): 450 if f.endswith(".py"): 451 self.command.append("--testmodule=%s" % f) 452 else: 453 self.command.extend(self.tests) 454 log.msg("Trial.start: command is", self.command) 455 456 # if our slave is too old to understand logfiles=, fetch them 457 # manually. This is a fallback for the Twisted buildbot and some old 458 # buildslaves. 459 self._needToPullTestDotLog = False 460 if self.slaveVersionIsOlderThan("shell", "2.1"): 461 log.msg("Trial: buildslave %s is too old to accept logfiles=" % 462 self.getSlaveName()) 463 log.msg(" falling back to 'cat _trial_temp/test.log' instead") 464 self.logfiles = {} 465 self._needToPullTestDotLog = True 466 467 ShellCommand.start(self)

468 469

470 - def commandComplete(self, cmd):

471 if not self._needToPullTestDotLog: 472 return self._gotTestDotLog(cmd) 473 474 # if the buildslave was too old, pull test.log now 475 catcmd = ["cat", "_trial_temp/test.log"] 476 c2 = RemoteShellCommand(command=catcmd, workdir=self.workdir) 477 loog = self.addLog("test.log") 478 c2.useLog(loog, True, logfileName="stdio") 479 self.cmd = c2 # to allow interrupts 480 d = c2.run(self, self.remote) 481 d.addCallback(lambda res: self._gotTestDotLog(cmd)) 482 return d

483

484 - def rtext(self, fmt='%s'):

485 if self.reactor: 486 rtext = fmt % self.reactor 487 return rtext.replace("reactor", "") 488 return ""

489

490 - def _gotTestDotLog(self, cmd):

491 # figure out all status, then let the various hook functions return 492 # different pieces of it 493 494 # 'cmd' is the original trial command, so cmd.logs['stdio'] is the 495 # trial output. We don't have access to test.log from here. 496 output = cmd.logs['stdio'].getText() 497 counts = countFailedTests(output) 498 499 total = counts['total'] 500 failures, errors = counts['failures'], counts['errors'] 501 parsed = (total != None) 502 text = [] 503 text2 = "" 504 505 if cmd.rc == 0: 506 if parsed: 507 results = SUCCESS 508 if total: 509 text += ["%d %s" % \ 510 (total, 511 total == 1 and "test" or "tests"), 512 "passed"] 513 else: 514 text += ["no tests", "run"] 515 else: 516 results = FAILURE 517 text += ["testlog", "unparseable"] 518 text2 = "tests" 519 else: 520 # something failed 521 results = FAILURE 522 if parsed: 523 text.append("tests") 524 if failures: 525 text.append("%d %s" % \ 526 (failures, 527 failures == 1 and "failure" or "failures")) 528 if errors: 529 text.append("%d %s" % \ 530 (errors, 531 errors == 1 and "error" or "errors")) 532 count = failures + errors 533 text2 = "%d tes%s" % (count, (count == 1 and 't' or 'ts')) 534 else: 535 text += ["tests", "failed"] 536 text2 = "tests" 537 538 if counts['skips']: 539 text.append("%d %s" % \ 540 (counts['skips'], 541 counts['skips'] == 1 and "skip" or "skips")) 542 if counts['expectedFailures']: 543 text.append("%d %s" % \ 544 (counts['expectedFailures'], 545 counts['expectedFailures'] == 1 and "todo" 546 or "todos")) 547 if 0: # TODO 548 results = WARNINGS 549 if not text2: 550 text2 = "todo" 551 552 if 0: 553 # ignore unexpectedSuccesses for now, but it should really mark 554 # the build WARNING 555 if counts['unexpectedSuccesses']: 556 text.append("%d surprises" % counts['unexpectedSuccesses']) 557 results = WARNINGS 558 if not text2: 559 text2 = "tests" 560 561 if self.reactor: 562 text.append(self.rtext('(%s)')) 563 if text2: 564 text2 = "%s %s" % (text2, self.rtext('(%s)')) 565 566 self.results = results 567 self.text = text 568 self.text2 = [text2]

569

570 - def addTestResult(self, testname, results, text, tlog):

571 if self.reactor is not None: 572 testname = (self.reactor,) + testname 573 tr = builder.TestResult(testname, results, text, logs={'log': tlog}) 574 #self.step_status.build.addTestResult(tr) 575 self.build.build_status.addTestResult(tr)

576

577 - def createSummary(self, loog):

578 output = loog.getText() 579 problems = "" 580 sio = StringIO.StringIO(output) 581 warnings = {} 582 while 1: 583 line = sio.readline() 584 if line == "": 585 break 586 if line.find(" exceptions.DeprecationWarning: ") != -1: 587 # no source 588 warning = line # TODO: consider stripping basedir prefix here 589 warnings[warning] = warnings.get(warning, 0) + 1 590 elif (line.find(" DeprecationWarning: ") != -1 or 591 line.find(" UserWarning: ") != -1): 592 # next line is the source 593 warning = line + sio.readline() 594 warnings[warning] = warnings.get(warning, 0) + 1 595 elif line.find("Warning: ") != -1: 596 warning = line 597 warnings[warning] = warnings.get(warning, 0) + 1 598 599 if line.find("=" * 60) == 0 or line.find("-" * 60) == 0: 600 problems += line 601 problems += sio.read() 602 break 603 604 if problems: 605 self.addCompleteLog("problems", problems) 606 # now parse the problems for per-test results 607 pio = StringIO.StringIO(problems) 608 pio.readline() # eat the first separator line 609 testname = None 610 done = False 611 while not done: 612 while 1: 613 line = pio.readline() 614 if line == "": 615 done = True 616 break 617 if line.find("=" * 60) == 0: 618 break 619 if line.find("-" * 60) == 0: 620 # the last case has --- as a separator before the 621 # summary counts are printed 622 done = True 623 break 624 if testname is None: 625 # the first line after the === is like: 626 # EXPECTED FAILURE: testLackOfTB (twisted.test.test_failure.FailureTestCase) 627 # SKIPPED: testRETR (twisted.test.test_ftp.TestFTPServer) 628 # FAILURE: testBatchFile (twisted.conch.test.test_sftp.TestOurServerBatchFile) 629 r = re.search(r'^([^:]+): (\w+) $([\w\.]+)$', line) 630 if not r: 631 # TODO: cleanup, if there are no problems, 632 # we hit here 633 continue 634 result, name, case = r.groups() 635 testname = tuple(case.split(".") + [name]) 636 results = {'SKIPPED': SKIPPED, 637 'EXPECTED FAILURE': SUCCESS, 638 'UNEXPECTED SUCCESS': WARNINGS, 639 'FAILURE': FAILURE, 640 'ERROR': FAILURE, 641 'SUCCESS': SUCCESS, # not reported 642 }.get(result, WARNINGS) 643 text = result.lower().split() 644 loog = line 645 # the next line is all dashes 646 loog += pio.readline() 647 else: 648 # the rest goes into the log 649 loog += line 650 if testname: 651 self.addTestResult(testname, results, text, loog) 652 testname = None 653 654 if warnings: 655 lines = warnings.keys() 656 lines.sort() 657 self.addCompleteLog("warnings", "".join(lines))

658

659 - def evaluateCommand(self, cmd):

660 return self.results

661

662 - def getText(self, cmd, results):

663 return self.text

664 - def getText2(self, cmd, results):

665 return self.text2

Source Code for Module buildbot.steps.python_twisted