buildbot.steps.shell

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 17 import re 18 from twisted.python import log, failure 19 from twisted.spread import pb 20 from buildbot.process import buildstep 21 from buildbot.status.results import SUCCESS, WARNINGS, FAILURE 22 from buildbot.status.logfile import STDOUT, STDERR 23 24 # for existing configurations that import WithProperties from here. We like 25 # to move this class around just to keep our readers guessing. 26 from buildbot.process.properties import WithProperties 27 _hush_pyflakes = [WithProperties] 28 del _hush_pyflakes 29

30 -class ShellCommand(buildstep.LoggingBuildStep):

31 """I run a single shell command on the buildslave. I return FAILURE if 32 the exit code of that command is non-zero, SUCCESS otherwise. To change 33 this behavior, override my .evaluateCommand method. 34 35 By default, a failure of this step will mark the whole build as FAILURE. 36 To override this, give me an argument of flunkOnFailure=False . 37 38 I create a single Log named 'log' which contains the output of the 39 command. To create additional summary Logs, override my .createSummary 40 method. 41 42 The shell command I run (a list of argv strings) can be provided in 43 several ways: 44 - a class-level .command attribute 45 - a command= parameter to my constructor (overrides .command) 46 - set explicitly with my .setCommand() method (overrides both) 47 48 @ivar command: a list of renderable objects (typically strings or 49 WithProperties instances). This will be used by start() 50 to create a RemoteShellCommand instance. 51 52 @ivar logfiles: a dict mapping log NAMEs to workdir-relative FILENAMEs 53 of their corresponding logfiles. The contents of the file 54 named FILENAME will be put into a LogFile named NAME, ina 55 something approximating real-time. (note that logfiles= 56 is actually handled by our parent class LoggingBuildStep) 57 58 @ivar lazylogfiles: Defaults to False. If True, logfiles will be tracked 59 `lazily', meaning they will only be added when and if 60 they are written to. Empty or nonexistent logfiles 61 will be omitted. (Also handled by class 62 LoggingBuildStep.) 63 64 """ 65 66 name = "shell" 67 renderables = [ 'description', 'descriptionDone', 'slaveEnvironment', 'remote_kwargs', 'command' ] 68 description = None # set this to a list of short strings to override 69 descriptionDone = None # alternate description when the step is complete 70 command = None # set this to a command, or set in kwargs 71 # logfiles={} # you can also set 'logfiles' to a dictionary, and it 72 # will be merged with any logfiles= argument passed in 73 # to __init__ 74 75 # override this on a specific ShellCommand if you want to let it fail 76 # without dooming the entire build to a status of FAILURE 77 flunkOnFailure = True 78

79 - def __init__(self, workdir=None, 80 description=None, descriptionDone=None, 81 command=None, 82 usePTY="slave-config", 83 **kwargs):

84 # most of our arguments get passed through to the RemoteShellCommand 85 # that we create, but first strip out the ones that we pass to 86 # BuildStep (like haltOnFailure and friends), and a couple that we 87 # consume ourselves. 88 89 if description: 90 self.description = description 91 if isinstance(self.description, str): 92 self.description = [self.description] 93 if descriptionDone: 94 self.descriptionDone = descriptionDone 95 if isinstance(self.descriptionDone, str): 96 self.descriptionDone = [self.descriptionDone] 97 if command: 98 self.setCommand(command) 99 100 # pull out the ones that LoggingBuildStep wants, then upcall 101 buildstep_kwargs = {} 102 for k in kwargs.keys()[:]: 103 if k in self.__class__.parms: 104 buildstep_kwargs[k] = kwargs[k] 105 del kwargs[k] 106 buildstep.LoggingBuildStep.__init__(self, **buildstep_kwargs) 107 self.addFactoryArguments(workdir=workdir, 108 description=description, 109 descriptionDone=descriptionDone, 110 command=command) 111 112 # everything left over goes to the RemoteShellCommand 113 kwargs['workdir'] = workdir # including a copy of 'workdir' 114 kwargs['usePTY'] = usePTY 115 self.remote_kwargs = kwargs 116 # we need to stash the RemoteShellCommand's args too 117 self.addFactoryArguments(**kwargs)

118

119 - def setBuild(self, build):

120 buildstep.LoggingBuildStep.setBuild(self, build) 121 # Set this here, so it gets rendered when we start the step 122 self.slaveEnvironment = self.build.slaveEnvironment

123

124 - def setStepStatus(self, step_status):

125 buildstep.LoggingBuildStep.setStepStatus(self, step_status)

126

127 - def setDefaultWorkdir(self, workdir):

128 rkw = self.remote_kwargs 129 rkw['workdir'] = rkw['workdir'] or workdir

130

131 - def getWorkdir(self):

132 """ 133 Get the current notion of the workdir. Note that this may change 134 between instantiation of the step and C{start}, as it is based on the 135 build's default workdir, and may even be C{None} before that point. 136 """ 137 return self.remote_kwargs['workdir']

138

139 - def setCommand(self, command):

140 self.command = command

141

142 - def describe(self, done=False):

143 """Return a list of short strings to describe this step, for the 144 status display. This uses the first few words of the shell command. 145 You can replace this by setting .description in your subclass, or by 146 overriding this method to describe the step better. 147 148 @type done: boolean 149 @param done: whether the command is complete or not, to improve the 150 way the command is described. C{done=False} is used 151 while the command is still running, so a single 152 imperfect-tense verb is appropriate ('compiling', 153 'testing', ...) C{done=True} is used when the command 154 has finished, and the default getText() method adds some 155 text, so a simple noun is appropriate ('compile', 156 'tests' ...) 157 """ 158 159 try: 160 if done and self.descriptionDone is not None: 161 return self.descriptionDone 162 if self.description is not None: 163 return self.description 164 165 # we may have no command if this is a step that sets its command 166 # name late in the game (e.g., in start()) 167 if not self.command: 168 return ["???"] 169 170 words = self.command 171 if isinstance(words, (str, unicode)): 172 words = words.split() 173 if len(words) < 1: 174 return ["???"] 175 if len(words) == 1: 176 return ["'%s'" % words[0]] 177 if len(words) == 2: 178 return ["'%s" % words[0], "%s'" % words[1]] 179 return ["'%s" % words[0], "%s" % words[1], "...'"] 180 except: 181 log.err(failure.Failure(), "Error describing step") 182 return ["???"]

183

184 - def setupEnvironment(self, cmd):

185 # merge in anything from Build.slaveEnvironment 186 # This can be set from a Builder-level environment, or from earlier 187 # BuildSteps. The latter method is deprecated and superceded by 188 # BuildProperties. 189 # Environment variables passed in by a BuildStep override 190 # those passed in at the Builder level. 191 slaveEnv = self.slaveEnvironment 192 if slaveEnv: 193 if cmd.args['env'] is None: 194 cmd.args['env'] = {} 195 fullSlaveEnv = slaveEnv.copy() 196 fullSlaveEnv.update(cmd.args['env']) 197 cmd.args['env'] = fullSlaveEnv

198 # note that each RemoteShellCommand gets its own copy of the 199 # dictionary, so we shouldn't be affecting anyone but ourselves. 200

201 - def buildCommandKwargs(self, warnings):

202 kwargs = buildstep.LoggingBuildStep.buildCommandKwargs(self) 203 kwargs.update(self.remote_kwargs) 204 kwargs['command'] = self.command 205 206 # check for the usePTY flag 207 if kwargs.has_key('usePTY') and kwargs['usePTY'] != 'slave-config': 208 if self.slaveVersionIsOlderThan("svn", "2.7"): 209 warnings.append("NOTE: slave does not allow master to override usePTY\n") 210 del kwargs['usePTY'] 211 212 return kwargs

213

214 - def start(self):

215 # this block is specific to ShellCommands. subclasses that don't need 216 # to set up an argv array, an environment, or extra logfiles= (like 217 # the Source subclasses) can just skip straight to startCommand() 218 219 warnings = [] 220 221 # create the actual RemoteShellCommand instance now 222 kwargs = self.buildCommandKwargs(warnings) 223 cmd = buildstep.RemoteShellCommand(**kwargs) 224 self.setupEnvironment(cmd) 225 226 self.startCommand(cmd, warnings)

227 228 229

230 -class TreeSize(ShellCommand):

231 name = "treesize" 232 command = ["du", "-s", "-k", "."] 233 description = "measuring tree size" 234 descriptionDone = "tree size measured" 235 kib = None 236

237 - def commandComplete(self, cmd):

238 out = cmd.logs['stdio'].getText() 239 m = re.search(r'^(\d+)', out) 240 if m: 241 self.kib = int(m.group(1)) 242 self.setProperty("tree-size-KiB", self.kib, "treesize")

243

244 - def evaluateCommand(self, cmd):

245 if cmd.rc != 0: 246 return FAILURE 247 if self.kib is None: 248 return WARNINGS # not sure how 'du' could fail, but whatever 249 return SUCCESS

250

251 - def getText(self, cmd, results):

252 if self.kib is not None: 253 return ["treesize", "%d KiB" % self.kib] 254 return ["treesize", "unknown"]

255 256

257 -class SetProperty(ShellCommand):

258 name = "setproperty" 259 renderables = [ 'property' ] 260

261 - def __init__(self, property=None, extract_fn=None, strip=True, **kwargs):

262 self.property = property 263 self.extract_fn = extract_fn 264 self.strip = strip 265 266 assert (property is not None) ^ (extract_fn is not None), \ 267 "Exactly one of property and extract_fn must be set" 268 269 ShellCommand.__init__(self, **kwargs) 270 271 self.addFactoryArguments(property=self.property) 272 self.addFactoryArguments(extract_fn=self.extract_fn) 273 self.addFactoryArguments(strip=self.strip) 274 275 self.property_changes = {}

276

277 - def commandComplete(self, cmd):

278 if self.property: 279 if cmd.rc != 0: 280 return 281 result = cmd.logs['stdio'].getText() 282 if self.strip: result = result.strip() 283 propname = self.property 284 self.setProperty(propname, result, "SetProperty Step") 285 self.property_changes[propname] = result 286 else: 287 log = cmd.logs['stdio'] 288 new_props = self.extract_fn(cmd.rc, 289 ''.join(log.getChunks([STDOUT], onlyText=True)), 290 ''.join(log.getChunks([STDERR], onlyText=True))) 291 for k,v in new_props.items(): 292 self.setProperty(k, v, "SetProperty Step") 293 self.property_changes = new_props

294

295 - def createSummary(self, log):

296 if self.property_changes: 297 props_set = [ "%s: %r" % (k,v) 298 for k,v in self.property_changes.items() ] 299 self.addCompleteLog('property changes', "\n".join(props_set))

300

301 - def getText(self, cmd, results):

302 if len(self.property_changes) > 1: 303 return [ "%d properties set" % len(self.property_changes) ] 304 elif len(self.property_changes) == 1: 305 return [ "property '%s' set" % self.property_changes.keys()[0] ] 306 else: 307 # let ShellCommand describe 308 return ShellCommand.getText(self, cmd, results)

309

310 -class Configure(ShellCommand):

311 312 name = "configure" 313 haltOnFailure = 1 314 flunkOnFailure = 1 315 description = ["configuring"] 316 descriptionDone = ["configure"] 317 command = ["./configure"]

318

319 -class StringFileWriter(pb.Referenceable):

320 """ 321 FileWriter class that just puts received data into a buffer. 322 323 Used to upload a file from slave for inline processing rather than 324 writing into a file on master. 325 """

326 - def __init__(self):

327 self.buffer = ""

328

329 - def remote_write(self, data):

330 self.buffer += data

331

332 - def remote_close(self):

333 pass

334

335 -class WarningCountingShellCommand(ShellCommand):

336 renderables = [ 'suppressionFile' ] 337 338 warnCount = 0 339 warningPattern = '.*warning[: ].*' 340 # The defaults work for GNU Make. 341 directoryEnterPattern = (u"make.*: Entering directory " 342 u"[\u2019\"`'](.*)[\u2019'`\"]") 343 directoryLeavePattern = "make.*: Leaving directory" 344 suppressionFile = None 345 346 commentEmptyLineRe = re.compile(r"^\s*(\#.*)?$") 347 suppressionLineRe = re.compile(r"^\s*(.+?)\s*:\s*(.+?)\s*(?:[:]\s*([0-9]+)(?:-([0-9]+))?\s*)?$") 348

349 - def __init__(self, 350 warningPattern=None, warningExtractor=None, maxWarnCount=None, 351 directoryEnterPattern=None, directoryLeavePattern=None, 352 suppressionFile=None, **kwargs):

353 # See if we've been given a regular expression to use to match 354 # warnings. If not, use a default that assumes any line with "warning" 355 # present is a warning. This may lead to false positives in some cases. 356 if warningPattern: 357 self.warningPattern = warningPattern 358 if directoryEnterPattern: 359 self.directoryEnterPattern = directoryEnterPattern 360 if directoryLeavePattern: 361 self.directoryLeavePattern = directoryLeavePattern 362 if suppressionFile: 363 self.suppressionFile = suppressionFile 364 if warningExtractor: 365 self.warningExtractor = warningExtractor 366 else: 367 self.warningExtractor = WarningCountingShellCommand.warnExtractWholeLine 368 self.maxWarnCount = maxWarnCount 369 370 # And upcall to let the base class do its work 371 ShellCommand.__init__(self, **kwargs) 372 373 self.addFactoryArguments(warningPattern=warningPattern, 374 directoryEnterPattern=directoryEnterPattern, 375 directoryLeavePattern=directoryLeavePattern, 376 warningExtractor=warningExtractor, 377 maxWarnCount=maxWarnCount, 378 suppressionFile=suppressionFile) 379 self.suppressions = [] 380 self.directoryStack = []

381

382 - def addSuppression(self, suppressionList):

383 """ 384 This method can be used to add patters of warnings that should 385 not be counted. 386 387 It takes a single argument, a list of patterns. 388 389 Each pattern is a 4-tuple (FILE-RE, WARN-RE, START, END). 390 391 FILE-RE is a regular expression (string or compiled regexp), or None. 392 If None, the pattern matches all files, else only files matching the 393 regexp. If directoryEnterPattern is specified in the class constructor, 394 matching is against the full path name, eg. src/main.c. 395 396 WARN-RE is similarly a regular expression matched against the 397 text of the warning, or None to match all warnings. 398 399 START and END form an inclusive line number range to match against. If 400 START is None, there is no lower bound, similarly if END is none there 401 is no upper bound.""" 402 403 for fileRe, warnRe, start, end in suppressionList: 404 if fileRe != None and isinstance(fileRe, basestring): 405 fileRe = re.compile(fileRe) 406 if warnRe != None and isinstance(warnRe, basestring): 407 warnRe = re.compile(warnRe) 408 self.suppressions.append((fileRe, warnRe, start, end))

409

410 - def warnExtractWholeLine(self, line, match):

411 """ 412 Extract warning text as the whole line. 413 No file names or line numbers.""" 414 return (None, None, line)

415

416 - def warnExtractFromRegexpGroups(self, line, match):

417 """ 418 Extract file name, line number, and warning text as groups (1,2,3) 419 of warningPattern match.""" 420 file = match.group(1) 421 lineNo = match.group(2) 422 if lineNo != None: 423 lineNo = int(lineNo) 424 text = match.group(3) 425 return (file, lineNo, text)

426

427 - def maybeAddWarning(self, warnings, line, match):

428 if self.suppressions: 429 (file, lineNo, text) = self.warningExtractor(self, line, match) 430 lineNo = lineNo and int(lineNo) 431 432 if file != None and file != "" and self.directoryStack: 433 currentDirectory = '/'.join(self.directoryStack) 434 if currentDirectory != None and currentDirectory != "": 435 file = "%s/%s" % (currentDirectory, file) 436 437 # Skip adding the warning if any suppression matches. 438 for fileRe, warnRe, start, end in self.suppressions: 439 if not (file == None or fileRe == None or fileRe.match(file)): 440 continue 441 if not (warnRe == None or warnRe.search(text)): 442 continue 443 if not ((start == None and end == None) or 444 (lineNo != None and start <= lineNo and end >= lineNo)): 445 continue 446 return 447 448 warnings.append(line) 449 self.warnCount += 1

450

451 - def start(self):

452 if self.suppressionFile == None: 453 return ShellCommand.start(self) 454 455 self.myFileWriter = StringFileWriter() 456 457 args = { 458 'slavesrc': self.suppressionFile, 459 'workdir': self.getWorkdir(), 460 'writer': self.myFileWriter, 461 'maxsize': None, 462 'blocksize': 32*1024, 463 } 464 cmd = buildstep.RemoteCommand('uploadFile', args, ignore_updates=True) 465 d = self.runCommand(cmd) 466 d.addCallback(self.uploadDone) 467 d.addErrback(self.failed)

468

469 - def uploadDone(self, dummy):

470 lines = self.myFileWriter.buffer.split("\n") 471 del(self.myFileWriter) 472 473 list = [] 474 for line in lines: 475 if self.commentEmptyLineRe.match(line): 476 continue 477 match = self.suppressionLineRe.match(line) 478 if (match): 479 file, test, start, end = match.groups() 480 if (end != None): 481 end = int(end) 482 if (start != None): 483 start = int(start) 484 if end == None: 485 end = start 486 list.append((file, test, start, end)) 487 488 self.addSuppression(list) 489 return ShellCommand.start(self)

490

491 - def createSummary(self, log):

492 """ 493 Match log lines against warningPattern. 494 495 Warnings are collected into another log for this step, and the 496 build-wide 'warnings-count' is updated.""" 497 498 self.warnCount = 0 499 500 # Now compile a regular expression from whichever warning pattern we're 501 # using 502 wre = self.warningPattern 503 if isinstance(wre, str): 504 wre = re.compile(wre) 505 506 directoryEnterRe = self.directoryEnterPattern 507 if (directoryEnterRe != None 508 and isinstance(directoryEnterRe, basestring)): 509 directoryEnterRe = re.compile(directoryEnterRe) 510 511 directoryLeaveRe = self.directoryLeavePattern 512 if (directoryLeaveRe != None 513 and isinstance(directoryLeaveRe, basestring)): 514 directoryLeaveRe = re.compile(directoryLeaveRe) 515 516 # Check if each line in the output from this command matched our 517 # warnings regular expressions. If did, bump the warnings count and 518 # add the line to the collection of lines with warnings 519 warnings = [] 520 # TODO: use log.readlines(), except we need to decide about stdout vs 521 # stderr 522 for line in log.getText().split("\n"): 523 if directoryEnterRe: 524 match = directoryEnterRe.search(line) 525 if match: 526 self.directoryStack.append(match.group(1)) 527 continue 528 if (directoryLeaveRe and 529 self.directoryStack and 530 directoryLeaveRe.search(line)): 531 self.directoryStack.pop() 532 continue 533 534 match = wre.match(line) 535 if match: 536 self.maybeAddWarning(warnings, line, match) 537 538 # If there were any warnings, make the log if lines with warnings 539 # available 540 if self.warnCount: 541 self.addCompleteLog("warnings (%d)" % self.warnCount, 542 "\n".join(warnings) + "\n") 543 544 warnings_stat = self.step_status.getStatistic('warnings', 0) 545 self.step_status.setStatistic('warnings', warnings_stat + self.warnCount) 546 547 try: 548 old_count = self.getProperty("warnings-count") 549 except KeyError: 550 old_count = 0 551 self.setProperty("warnings-count", old_count + self.warnCount, "WarningCountingShellCommand")

552 553

554 - def evaluateCommand(self, cmd):

555 if ( cmd.rc != 0 or 556 ( self.maxWarnCount != None and self.warnCount > self.maxWarnCount ) ): 557 return FAILURE 558 if self.warnCount: 559 return WARNINGS 560 return SUCCESS

561 562

563 -class Compile(WarningCountingShellCommand):

564 565 name = "compile" 566 haltOnFailure = 1 567 flunkOnFailure = 1 568 description = ["compiling"] 569 descriptionDone = ["compile"] 570 command = ["make", "all"]

571

572 -class Test(WarningCountingShellCommand):

573 574 name = "test" 575 warnOnFailure = 1 576 description = ["testing"] 577 descriptionDone = ["test"] 578 command = ["make", "test"] 579

580 - def setTestResults(self, total=0, failed=0, passed=0, warnings=0):

581 """ 582 Called by subclasses to set the relevant statistics; this actually 583 adds to any statistics already present 584 """ 585 total += self.step_status.getStatistic('tests-total', 0) 586 self.step_status.setStatistic('tests-total', total) 587 failed += self.step_status.getStatistic('tests-failed', 0) 588 self.step_status.setStatistic('tests-failed', failed) 589 warnings += self.step_status.getStatistic('tests-warnings', 0) 590 self.step_status.setStatistic('tests-warnings', warnings) 591 passed += self.step_status.getStatistic('tests-passed', 0) 592 self.step_status.setStatistic('tests-passed', passed)

593

594 - def describe(self, done=False):

595 description = WarningCountingShellCommand.describe(self, done) 596 if done: 597 description = description[:] # make a private copy 598 if self.step_status.hasStatistic('tests-total'): 599 total = self.step_status.getStatistic("tests-total", 0) 600 failed = self.step_status.getStatistic("tests-failed", 0) 601 passed = self.step_status.getStatistic("tests-passed", 0) 602 warnings = self.step_status.getStatistic("tests-warnings", 0) 603 if not total: 604 total = failed + passed + warnings 605 606 if total: 607 description.append('%d tests' % total) 608 if passed: 609 description.append('%d passed' % passed) 610 if warnings: 611 description.append('%d warnings' % warnings) 612 if failed: 613 description.append('%d failed' % failed) 614 return description

615

616 -class PerlModuleTest(Test):

617 command=["prove", "--lib", "lib", "-r", "t"] 618 total = 0 619

620 - def evaluateCommand(self, cmd):

621 # Get stdio, stripping pesky newlines etc. 622 lines = map( 623 lambda line : line.replace('\r\n','').replace('\r','').replace('\n',''), 624 self.getLog('stdio').readlines() 625 ) 626 627 total = 0 628 passed = 0 629 failed = 0 630 rc = SUCCESS 631 if cmd.rc > 0: 632 rc = FAILURE 633 634 # New version of Test::Harness? 635 if "Test Summary Report" in lines: 636 test_summary_report_index = lines.index("Test Summary Report") 637 del lines[0:test_summary_report_index + 2] 638 639 re_test_result = re.compile("^Result: (PASS|FAIL)$|Tests: \d+ Failed: (\d+)\)|Files=\d+, Tests=(\d+)") 640 641 mos = map(lambda line: re_test_result.search(line), lines) 642 test_result_lines = [mo.groups() for mo in mos if mo] 643 644 for line in test_result_lines: 645 if line[0] == 'FAIL': 646 rc = FAILURE 647 648 if line[1]: 649 failed += int(line[1]) 650 if line[2]: 651 total = int(line[2]) 652 653 else: # Nope, it's the old version 654 re_test_result = re.compile("^(All tests successful)|(\d+)/(\d+) subtests failed|Files=\d+, Tests=(\d+),") 655 656 mos = map(lambda line: re_test_result.search(line), lines) 657 test_result_lines = [mo.groups() for mo in mos if mo] 658 659 if test_result_lines: 660 test_result_line = test_result_lines[0] 661 662 success = test_result_line[0] 663 664 if success: 665 failed = 0 666 667 test_totals_line = test_result_lines[1] 668 total_str = test_totals_line[3] 669 else: 670 failed_str = test_result_line[1] 671 failed = int(failed_str) 672 673 total_str = test_result_line[2] 674 675 rc = FAILURE 676 677 total = int(total_str) 678 679 warnings = 0 680 if self.warningPattern: 681 wre = self.warningPattern 682 if isinstance(wre, str): 683 wre = re.compile(wre) 684 685 warnings = len([l for l in lines if wre.search(l)]) 686 687 # Because there are two paths that are used to determine 688 # the success/fail result, I have to modify it here if 689 # there were warnings. 690 if rc == SUCCESS and warnings: 691 rc = WARNINGS 692 693 if total: 694 passed = total - failed 695 696 self.setTestResults(total=total, failed=failed, passed=passed, 697 warnings=warnings) 698 699 return rc

700

Source Code for Module buildbot.steps.shell