buildbot.steps.python

168 """ 169 There are some class attributes which may be usefully overridden 170 by subclasses. 'trialMode' and 'trialArgs' can influence the trial 171 command line. 172 """ 173 174 name = "trial" 175 progressMetrics = ('output', 'tests', 'test.log') 176 # note: the slash only works on unix buildslaves, of course, but we have 177 # no way to know what the buildslave uses as a separator. 178 # TODO: figure out something clever. 179 logfiles = {"test.log": "_trial_temp/test.log"} 180 # we use test.log to track Progress at the end of __init__() 181 182 flunkOnFailure = True 183 python = None 184 trial = "trial" 185 trialMode = ["--reporter=bwverbose"] # requires Twisted-2.1.0 or newer 186 # for Twisted-2.0.0 or 1.3.0, use ["-o"] instead 187 trialArgs = [] 188 testpath = UNSPECIFIED # required (but can be None) 189 testChanges = False # TODO: needs better name 190 recurse = False 191 reactor = None 192 randomly = False 193 tests = None # required 194

195 - def __init__(self, reactor=UNSPECIFIED, python=None, trial=None, 196 testpath=UNSPECIFIED, 197 tests=None, testChanges=None, 198 recurse=None, randomly=None, 199 trialMode=None, trialArgs=None, 200 **kwargs):

201 """ 202 @type testpath: string 203 @param testpath: use in PYTHONPATH when running the tests. If 204 None, do not set PYTHONPATH. Setting this to '.' will 205 cause the source files to be used in-place. 206 207 @type python: string (without spaces) or list 208 @param python: which python executable to use. Will form the start of 209 the argv array that will launch trial. If you use this, 210 you should set 'trial' to an explicit path (like 211 /usr/bin/trial or ./bin/trial). Defaults to None, which 212 leaves it out entirely (running 'trial args' instead of 213 'python ./bin/trial args'). Likely values are 'python', 214 ['python2.2'], ['python', '-Wall'], etc. 215 216 @type trial: string 217 @param trial: which 'trial' executable to run. 218 Defaults to 'trial', which will cause $PATH to be 219 searched and probably find /usr/bin/trial . If you set 220 'python', this should be set to an explicit path (because 221 'python2.3 trial' will not work). 222 223 @type trialMode: list of strings 224 @param trialMode: a list of arguments to pass to trial, specifically 225 to set the reporting mode. This defaults to ['-to'] 226 which means 'verbose colorless output' to the trial 227 that comes with Twisted-2.0.x and at least -2.1.0 . 228 Newer versions of Twisted may come with a trial 229 that prefers ['--reporter=bwverbose']. 230 231 @type trialArgs: list of strings 232 @param trialArgs: a list of arguments to pass to trial, available to 233 turn on any extra flags you like. Defaults to []. 234 235 @type tests: list of strings 236 @param tests: a list of test modules to run, like 237 ['twisted.test.test_defer', 'twisted.test.test_process']. 238 If this is a string, it will be converted into a one-item 239 list. 240 241 @type testChanges: boolean 242 @param testChanges: if True, ignore the 'tests' parameter and instead 243 ask the Build for all the files that make up the 244 Changes going into this build. Pass these filenames 245 to trial and ask it to look for test-case-name 246 tags, running just the tests necessary to cover the 247 changes. 248 249 @type recurse: boolean 250 @param recurse: If True, pass the --recurse option to trial, allowing 251 test cases to be found in deeper subdirectories of the 252 modules listed in 'tests'. This does not appear to be 253 necessary when using testChanges. 254 255 @type reactor: string 256 @param reactor: which reactor to use, like 'gtk' or 'java'. If not 257 provided, the Twisted's usual platform-dependent 258 default is used. 259 260 @type randomly: boolean 261 @param randomly: if True, add the --random=0 argument, which instructs 262 trial to run the unit tests in a random order each 263 time. This occasionally catches problems that might be 264 masked when one module always runs before another 265 (like failing to make registerAdapter calls before 266 lookups are done). 267 268 @type kwargs: dict 269 @param kwargs: parameters. The following parameters are inherited from 270 L{ShellCommand} and may be useful to set: workdir, 271 haltOnFailure, flunkOnWarnings, flunkOnFailure, 272 warnOnWarnings, warnOnFailure, want_stdout, want_stderr, 273 timeout. 274 """ 275 ShellCommand.__init__(self, **kwargs) 276 self.addFactoryArguments(reactor=reactor, 277 python=python, 278 trial=trial, 279 testpath=testpath, 280 tests=tests, 281 testChanges=testChanges, 282 recurse=recurse, 283 randomly=randomly, 284 trialMode=trialMode, 285 trialArgs=trialArgs, 286 ) 287 288 if python: 289 self.python = python 290 if self.python is not None: 291 if type(self.python) is str: 292 self.python = [self.python] 293 for s in self.python: 294 if " " in s: 295 # this is not strictly an error, but I suspect more 296 # people will accidentally try to use python="python2.3 297 # -Wall" than will use embedded spaces in a python flag 298 log.msg("python= component '%s' has spaces") 299 log.msg("To add -Wall, use python=['python', '-Wall']") 300 why = "python= value has spaces, probably an error" 301 raise ValueError(why) 302 303 if trial: 304 self.trial = trial 305 if " " in self.trial: 306 raise ValueError("trial= value has spaces") 307 if trialMode is not None: 308 self.trialMode = trialMode 309 if trialArgs is not None: 310 self.trialArgs = trialArgs 311 312 if testpath is not UNSPECIFIED: 313 self.testpath = testpath 314 if self.testpath is UNSPECIFIED: 315 raise ValueError("You must specify testpath= (it can be None)") 316 assert isinstance(self.testpath, str) or self.testpath is None 317 318 if reactor is not UNSPECIFIED: 319 self.reactor = reactor 320 321 if tests is not None: 322 self.tests = tests 323 if type(self.tests) is str: 324 self.tests = [self.tests] 325 if testChanges is not None: 326 self.testChanges = testChanges 327 #self.recurse = True # not sure this is necessary 328 329 if not self.testChanges and self.tests is None: 330 raise ValueError("Must either set testChanges= or provide tests=") 331 332 if recurse is not None: 333 self.recurse = recurse 334 if randomly is not None: 335 self.randomly = randomly 336 337 # build up most of the command, then stash it until start() 338 command = [] 339 if self.python: 340 command.extend(self.python) 341 command.append(self.trial) 342 command.extend(self.trialMode) 343 if self.recurse: 344 command.append("--recurse") 345 if self.reactor: 346 command.append("--reactor=%s" % reactor) 347 if self.randomly: 348 command.append("--random=0") 349 command.extend(self.trialArgs) 350 self.command = command 351 352 if self.reactor: 353 self.description = ["testing", "(%s)" % self.reactor] 354 self.descriptionDone = ["tests"] 355 # commandComplete adds (reactorname) to self.text 356 else: 357 self.description = ["testing"] 358 self.descriptionDone = ["tests"] 359 360 # this counter will feed Progress along the 'test cases' metric 361 self.addLogObserver('stdio', TrialTestCaseCounter()) 362 # this one just measures bytes of output in _trial_temp/test.log 363 self.addLogObserver('test.log', OutputProgressObserver('test.log'))

364

365 - def setupEnvironment(self, cmd):

366 ShellCommand.setupEnvironment(self, cmd) 367 if self.testpath != None: 368 e = cmd.args['env'] 369 if e is None: 370 cmd.args['env'] = {'PYTHONPATH': self.testpath} 371 else: 372 # TODO: somehow, each build causes another copy of 373 # self.testpath to get prepended 374 if e.get('PYTHONPATH', "") == "": 375 e['PYTHONPATH'] = self.testpath 376 else: 377 e['PYTHONPATH'] = self.testpath + ":" + e['PYTHONPATH'] 378 try: 379 p = cmd.args['env']['PYTHONPATH'] 380 if type(p) is not str: 381 log.msg("hey, not a string:", p) 382 assert False 383 except (KeyError, TypeError): 384 # KeyError if args doesn't have ['env'] 385 # KeyError if args['env'] doesn't have ['PYTHONPATH'] 386 # TypeError if args is None 387 pass

388

389 - def start(self):

390 # now that self.build.allFiles() is nailed down, finish building the 391 # command 392 if self.testChanges: 393 for f in self.build.allFiles(): 394 if f.endswith(".py"): 395 self.command.append("--testmodule=%s" % f) 396 else: 397 self.command.extend(self.tests) 398 log.msg("Trial.start: command is", self.command) 399 400 # if our slave is too old to understand logfiles=, fetch them 401 # manually. This is a fallback for the Twisted buildbot and some old 402 # buildslaves. 403 self._needToPullTestDotLog = False 404 if self.slaveVersionIsOlderThan("shell", "2.1"): 405 log.msg("Trial: buildslave %s is too old to accept logfiles=" % 406 self.getSlaveName()) 407 log.msg(" falling back to 'cat _trial_temp/test.log' instead") 408 self.logfiles = {} 409 self._needToPullTestDotLog = True 410 411 ShellCommand.start(self)

412 413

414 - def commandComplete(self, cmd):

415 if not self._needToPullTestDotLog: 416 return self._gotTestDotLog(cmd) 417 418 # if the buildslave was too old, pull test.log now 419 catcmd = ["cat", "_trial_temp/test.log"] 420 c2 = RemoteShellCommand(command=catcmd, workdir=self.workdir) 421 loog = self.addLog("test.log") 422 c2.useLog(loog, True, logfileName="stdio") 423 self.cmd = c2 # to allow interrupts 424 d = c2.run(self, self.remote) 425 d.addCallback(lambda res: self._gotTestDotLog(cmd)) 426 return d

427

428 - def rtext(self, fmt='%s'):

429 if self.reactor: 430 rtext = fmt % self.reactor 431 return rtext.replace("reactor", "") 432 return ""

433

434 - def _gotTestDotLog(self, cmd):

435 # figure out all status, then let the various hook functions return 436 # different pieces of it 437 438 # 'cmd' is the original trial command, so cmd.logs['stdio'] is the 439 # trial output. We don't have access to test.log from here. 440 output = cmd.logs['stdio'].getText() 441 counts = countFailedTests(output) 442 443 total = counts['total'] 444 failures, errors = counts['failures'], counts['errors'] 445 parsed = (total != None) 446 text = [] 447 text2 = "" 448 449 if cmd.rc == 0: 450 if parsed: 451 results = SUCCESS 452 if total: 453 text += ["%d %s" % \ 454 (total, 455 total == 1 and "test" or "tests"), 456 "passed"] 457 else: 458 text += ["no tests", "run"] 459 else: 460 results = FAILURE 461 text += ["testlog", "unparseable"] 462 text2 = "tests" 463 else: 464 # something failed 465 results = FAILURE 466 if parsed: 467 text.append("tests") 468 if failures: 469 text.append("%d %s" % \ 470 (failures, 471 failures == 1 and "failure" or "failures")) 472 if errors: 473 text.append("%d %s" % \ 474 (errors, 475 errors == 1 and "error" or "errors")) 476 count = failures + errors 477 text2 = "%d tes%s" % (count, (count == 1 and 't' or 'ts')) 478 else: 479 text += ["tests", "failed"] 480 text2 = "tests" 481 482 if counts['skips']: 483 text.append("%d %s" % \ 484 (counts['skips'], 485 counts['skips'] == 1 and "skip" or "skips")) 486 if counts['expectedFailures']: 487 text.append("%d %s" % \ 488 (counts['expectedFailures'], 489 counts['expectedFailures'] == 1 and "todo" 490 or "todos")) 491 if 0: # TODO 492 results = WARNINGS 493 if not text2: 494 text2 = "todo" 495 496 if 0: 497 # ignore unexpectedSuccesses for now, but it should really mark 498 # the build WARNING 499 if counts['unexpectedSuccesses']: 500 text.append("%d surprises" % counts['unexpectedSuccesses']) 501 results = WARNINGS 502 if not text2: 503 text2 = "tests" 504 505 if self.reactor: 506 text.append(self.rtext('(%s)')) 507 if text2: 508 text2 = "%s %s" % (text2, self.rtext('(%s)')) 509 510 self.results = results 511 self.text = text 512 self.text2 = [text2]

513

514 - def addTestResult(self, testname, results, text, tlog):

515 if self.reactor is not None: 516 testname = (self.reactor,) + testname 517 tr = builder.TestResult(testname, results, text, logs={'log': tlog}) 518 #self.step_status.build.addTestResult(tr) 519 self.build.build_status.addTestResult(tr)

520

521 - def createSummary(self, loog):

522 output = loog.getText() 523 problems = "" 524 sio = StringIO.StringIO(output) 525 warnings = {} 526 while 1: 527 line = sio.readline() 528 if line == "": 529 break 530 if line.find(" exceptions.DeprecationWarning: ") != -1: 531 # no source 532 warning = line # TODO: consider stripping basedir prefix here 533 warnings[warning] = warnings.get(warning, 0) + 1 534 elif (line.find(" DeprecationWarning: ") != -1 or 535 line.find(" UserWarning: ") != -1): 536 # next line is the source 537 warning = line + sio.readline() 538 warnings[warning] = warnings.get(warning, 0) + 1 539 elif line.find("Warning: ") != -1: 540 warning = line 541 warnings[warning] = warnings.get(warning, 0) + 1 542 543 if line.find("=" * 60) == 0 or line.find("-" * 60) == 0: 544 problems += line 545 problems += sio.read() 546 break 547 548 if problems: 549 self.addCompleteLog("problems", problems) 550 # now parse the problems for per-test results 551 pio = StringIO.StringIO(problems) 552 pio.readline() # eat the first separator line 553 testname = None 554 done = False 555 while not done: 556 while 1: 557 line = pio.readline() 558 if line == "": 559 done = True 560 break 561 if line.find("=" * 60) == 0: 562 break 563 if line.find("-" * 60) == 0: 564 # the last case has --- as a separator before the 565 # summary counts are printed 566 done = True 567 break 568 if testname is None: 569 # the first line after the === is like: 570 # EXPECTED FAILURE: testLackOfTB (twisted.test.test_failure.FailureTestCase) 571 # SKIPPED: testRETR (twisted.test.test_ftp.TestFTPServer) 572 # FAILURE: testBatchFile (twisted.conch.test.test_sftp.TestOurServerBatchFile) 573 r = re.search(r'^([^:]+): (\w+) $([\w\.]+)$', line) 574 if not r: 575 # TODO: cleanup, if there are no problems, 576 # we hit here 577 continue 578 result, name, case = r.groups() 579 testname = tuple(case.split(".") + [name]) 580 results = {'SKIPPED': SKIPPED, 581 'EXPECTED FAILURE': SUCCESS, 582 'UNEXPECTED SUCCESS': WARNINGS, 583 'FAILURE': FAILURE, 584 'ERROR': FAILURE, 585 'SUCCESS': SUCCESS, # not reported 586 }.get(result, WARNINGS) 587 text = result.lower().split() 588 loog = line 589 # the next line is all dashes 590 loog += pio.readline() 591 else: 592 # the rest goes into the log 593 loog += line 594 if testname: 595 self.addTestResult(testname, results, text, loog) 596 testname = None 597 598 if warnings: 599 lines = warnings.keys() 600 lines.sort() 601 self.addCompleteLog("warnings", "".join(lines))

602

603 - def evaluateCommand(self, cmd):

604 return self.results

605

606 - def getText(self, cmd, results):

607 return self.text

608 - def getText2(self, cmd, results):

609 return self.text2

Source Code for Module buildbot.steps.python_twisted