buildbot.process.buildstep

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 import re 17 18 from zope.interface import implements 19 from twisted.internet import reactor, defer, error 20 from twisted.protocols import basic 21 from twisted.spread import pb 22 from twisted.python import log, components 23 from twisted.python.failure import Failure 24 from twisted.web.util import formatFailure 25 from twisted.python.reflect import accumulateClassList 26 27 from buildbot import interfaces, locks, util, config 28 from buildbot.status import progress 29 from buildbot.status.results import SUCCESS, WARNINGS, FAILURE, SKIPPED, \ 30 EXCEPTION, RETRY, worst_status 31 from buildbot.process import metrics, properties

32 33 -class BuildStepFailed(Exception):

34 pass

35

36 -class RemoteCommand(pb.Referenceable):

37 38 # class-level unique identifier generator for command ids 39 _commandCounter = 0 40 41 active = False 42 rc = None 43 debug = False 44

45 - def __init__(self, remote_command, args, ignore_updates=False, collectStdout=False):

46 self.logs = {} 47 self.delayedLogs = {} 48 self._closeWhenFinished = {} 49 self.collectStdout = collectStdout 50 self.stdout = '' 51 52 self._startTime = None 53 self._remoteElapsed = None 54 self.remote_command = remote_command 55 self.args = args 56 self.ignore_updates = ignore_updates

57

58 - def __repr__(self):

59 return "<RemoteCommand '%s' at %d>" % (self.remote_command, id(self))

60

61 - def run(self, step, remote):

62 self.active = True 63 self.step = step 64 self.remote = remote 65 66 # generate a new command id 67 cmd_id = RemoteCommand._commandCounter 68 RemoteCommand._commandCounter += 1 69 self.commandID = "%d" % cmd_id 70 71 log.msg("%s: RemoteCommand.run [%s]" % (self, self.commandID)) 72 self.deferred = defer.Deferred() 73 74 d = defer.maybeDeferred(self._start) 75 76 # _finished is called with an error for unknown commands, errors 77 # that occur while the command is starting (including OSErrors in 78 # exec()), StaleBroker (when the connection was lost before we 79 # started), and pb.PBConnectionLost (when the slave isn't responding 80 # over this connection, perhaps it had a power failure, or NAT 81 # weirdness). If this happens, self.deferred is fired right away. 82 d.addErrback(self._finished) 83 84 # Connections which are lost while the command is running are caught 85 # when our parent Step calls our .lostRemote() method. 86 return self.deferred

87

88 - def useLog(self, loog, closeWhenFinished=False, logfileName=None):

89 assert interfaces.ILogFile.providedBy(loog) 90 if not logfileName: 91 logfileName = loog.getName() 92 assert logfileName not in self.logs 93 assert logfileName not in self.delayedLogs 94 self.logs[logfileName] = loog 95 self._closeWhenFinished[logfileName] = closeWhenFinished

96

97 - def useLogDelayed(self, logfileName, activateCallBack, closeWhenFinished=False):

98 assert logfileName not in self.logs 99 assert logfileName not in self.delayedLogs 100 self.delayedLogs[logfileName] = (activateCallBack, closeWhenFinished)

101

102 - def _start(self):

103 self.updates = {} 104 self._startTime = util.now() 105 106 # This method only initiates the remote command. 107 # We will receive remote_update messages as the command runs. 108 # We will get a single remote_complete when it finishes. 109 # We should fire self.deferred when the command is done. 110 d = self.remote.callRemote("startCommand", self, self.commandID, 111 self.remote_command, self.args) 112 return d

113

114 - def _finished(self, failure=None):

115 self.active = False 116 # call .remoteComplete. If it raises an exception, or returns the 117 # Failure that we gave it, our self.deferred will be errbacked. If 118 # it does not (either it ate the Failure or there the step finished 119 # normally and it didn't raise a new exception), self.deferred will 120 # be callbacked. 121 d = defer.maybeDeferred(self.remoteComplete, failure) 122 # arrange for the callback to get this RemoteCommand instance 123 # instead of just None 124 d.addCallback(lambda r: self) 125 # this fires the original deferred we returned from .run(), 126 # with self as the result, or a failure 127 d.addBoth(self.deferred.callback)

128

129 - def interrupt(self, why):

130 log.msg("RemoteCommand.interrupt", self, why) 131 if not self.active: 132 log.msg(" but this RemoteCommand is already inactive") 133 return defer.succeed(None) 134 if not self.remote: 135 log.msg(" but our .remote went away") 136 return defer.succeed(None) 137 if isinstance(why, Failure) and why.check(error.ConnectionLost): 138 log.msg("RemoteCommand.disconnect: lost slave") 139 self.remote = None 140 self._finished(why) 141 return defer.succeed(None) 142 143 # tell the remote command to halt. Returns a Deferred that will fire 144 # when the interrupt command has been delivered. 145 146 d = defer.maybeDeferred(self.remote.callRemote, "interruptCommand", 147 self.commandID, str(why)) 148 # the slave may not have remote_interruptCommand 149 d.addErrback(self._interruptFailed) 150 return d

151

152 - def _interruptFailed(self, why):

153 log.msg("RemoteCommand._interruptFailed", self) 154 # TODO: forcibly stop the Command now, since we can't stop it 155 # cleanly 156 return None

157

158 - def remote_update(self, updates):

159 """ 160 I am called by the slave's L{buildbot.slave.bot.SlaveBuilder} so 161 I can receive updates from the running remote command. 162 163 @type updates: list of [object, int] 164 @param updates: list of updates from the remote command 165 """ 166 self.buildslave.messageReceivedFromSlave() 167 max_updatenum = 0 168 for (update, num) in updates: 169 #log.msg("update[%d]:" % num) 170 try: 171 if self.active and not self.ignore_updates: 172 self.remoteUpdate(update) 173 except: 174 # log failure, terminate build, let slave retire the update 175 self._finished(Failure()) 176 # TODO: what if multiple updates arrive? should 177 # skip the rest but ack them all 178 if num > max_updatenum: 179 max_updatenum = num 180 return max_updatenum

181

182 - def remote_complete(self, failure=None):

183 """ 184 Called by the slave's L{buildbot.slave.bot.SlaveBuilder} to 185 notify me the remote command has finished. 186 187 @type failure: L{twisted.python.failure.Failure} or None 188 189 @rtype: None 190 """ 191 self.buildslave.messageReceivedFromSlave() 192 # call the real remoteComplete a moment later, but first return an 193 # acknowledgement so the slave can retire the completion message. 194 if self.active: 195 reactor.callLater(0, self._finished, failure) 196 return None

197

198 - def addStdout(self, data):

199 if 'stdio' in self.logs: 200 self.logs['stdio'].addStdout(data) 201 if self.collectStdout: 202 self.stdout += data

203

204 - def addStderr(self, data):

205 if 'stdio' in self.logs: 206 self.logs['stdio'].addStderr(data)

207

208 - def addHeader(self, data):

209 if 'stdio' in self.logs: 210 self.logs['stdio'].addHeader(data)

211

212 - def addToLog(self, logname, data):

213 # Activate delayed logs on first data. 214 if logname in self.delayedLogs: 215 (activateCallBack, closeWhenFinished) = self.delayedLogs[logname] 216 del self.delayedLogs[logname] 217 loog = activateCallBack(self) 218 self.logs[logname] = loog 219 self._closeWhenFinished[logname] = closeWhenFinished 220 221 if logname in self.logs: 222 self.logs[logname].addStdout(data) 223 else: 224 log.msg("%s.addToLog: no such log %s" % (self, logname))

225 226 @metrics.countMethod('RemoteCommand.remoteUpdate()')

227 - def remoteUpdate(self, update):

228 if self.debug: 229 for k,v in update.items(): 230 log.msg("Update[%s]: %s" % (k,v)) 231 if update.has_key('stdout'): 232 # 'stdout': data 233 self.addStdout(update['stdout']) 234 if update.has_key('stderr'): 235 # 'stderr': data 236 self.addStderr(update['stderr']) 237 if update.has_key('header'): 238 # 'header': data 239 self.addHeader(update['header']) 240 if update.has_key('log'): 241 # 'log': (logname, data) 242 logname, data = update['log'] 243 self.addToLog(logname, data) 244 if update.has_key('rc'): 245 rc = self.rc = update['rc'] 246 log.msg("%s rc=%s" % (self, rc)) 247 self.addHeader("program finished with exit code %d\n" % rc) 248 if update.has_key('elapsed'): 249 self._remoteElapsed = update['elapsed'] 250 251 # TODO: these should be handled at the RemoteCommand level 252 for k in update: 253 if k not in ('stdout', 'stderr', 'header', 'rc'): 254 if k not in self.updates: 255 self.updates[k] = [] 256 self.updates[k].append(update[k])

257

258 - def remoteComplete(self, maybeFailure):

259 if self._startTime and self._remoteElapsed: 260 delta = (util.now() - self._startTime) - self._remoteElapsed 261 metrics.MetricTimeEvent.log("RemoteCommand.overhead", delta) 262 263 for name,loog in self.logs.items(): 264 if self._closeWhenFinished[name]: 265 if maybeFailure: 266 loog.addHeader("\nremoteFailed: %s" % maybeFailure) 267 else: 268 log.msg("closing log %s" % loog) 269 loog.finish() 270 return maybeFailure

271 LoggedRemoteCommand = RemoteCommand

272 273 274 -class LogObserver:

275 implements(interfaces.ILogObserver) 276

277 - def setStep(self, step):

278 self.step = step

279

280 - def setLog(self, loog):

281 assert interfaces.IStatusLog.providedBy(loog) 282 loog.subscribe(self, True)

283

284 - def logChunk(self, build, step, log, channel, text):

285 if channel == interfaces.LOG_CHANNEL_STDOUT: 286 self.outReceived(text) 287 elif channel == interfaces.LOG_CHANNEL_STDERR: 288 self.errReceived(text)

289 290 # TODO: add a logEnded method? er, stepFinished? 291

292 - def outReceived(self, data):

293 """This will be called with chunks of stdout data. Override this in 294 your observer.""" 295 pass

296

297 - def errReceived(self, data):

298 """This will be called with chunks of stderr data. Override this in 299 your observer.""" 300 pass

301

302 303 -class LogLineObserver(LogObserver):

304 - def __init__(self):

305 self.stdoutParser = basic.LineOnlyReceiver() 306 self.stdoutParser.delimiter = "\n" 307 self.stdoutParser.lineReceived = self.outLineReceived 308 self.stdoutParser.transport = self # for the .disconnecting attribute 309 self.disconnecting = False 310 311 self.stderrParser = basic.LineOnlyReceiver() 312 self.stderrParser.delimiter = "\n" 313 self.stderrParser.lineReceived = self.errLineReceived 314 self.stderrParser.transport = self

315

316 - def setMaxLineLength(self, max_length):

317 """ 318 Set the maximum line length: lines longer than max_length are 319 dropped. Default is 16384 bytes. Use sys.maxint for effective 320 infinity. 321 """ 322 self.stdoutParser.MAX_LENGTH = max_length 323 self.stderrParser.MAX_LENGTH = max_length

324

325 - def outReceived(self, data):

326 self.stdoutParser.dataReceived(data)

327

328 - def errReceived(self, data):

329 self.stderrParser.dataReceived(data)

330

331 - def outLineReceived(self, line):

332 """This will be called with complete stdout lines (not including the 333 delimiter). Override this in your observer.""" 334 pass

335

336 - def errLineReceived(self, line):

337 """This will be called with complete lines of stderr (not including 338 the delimiter). Override this in your observer.""" 339 pass

340

341 342 -class RemoteShellCommand(RemoteCommand):

343 - def __init__(self, workdir, command, env=None, 344 want_stdout=1, want_stderr=1, 345 timeout=20*60, maxTime=None, logfiles={}, 346 usePTY="slave-config", logEnviron=True, 347 collectStdout=False, interruptSignal=None):

348 349 self.command = command # stash .command, set it later 350 if env is not None: 351 # avoid mutating the original master.cfg dictionary. Each 352 # ShellCommand gets its own copy, any start() methods won't be 353 # able to modify the original. 354 env = env.copy() 355 args = {'workdir': workdir, 356 'env': env, 357 'want_stdout': want_stdout, 358 'want_stderr': want_stderr, 359 'logfiles': logfiles, 360 'timeout': timeout, 361 'maxTime': maxTime, 362 'usePTY': usePTY, 363 'logEnviron': logEnviron, 364 } 365 if interruptSignal is not None: 366 args['interruptSignal'] = interruptSignal 367 RemoteCommand.__init__(self, "shell", args, collectStdout=collectStdout)

368

369 - def _start(self):

370 self.args['command'] = self.command 371 if self.remote_command == "shell": 372 # non-ShellCommand slavecommands are responsible for doing this 373 # fixup themselves 374 if self.step.slaveVersion("shell", "old") == "old": 375 self.args['dir'] = self.args['workdir'] 376 what = "command '%s' in dir '%s'" % (self.args['command'], 377 self.args['workdir']) 378 log.msg(what) 379 return RemoteCommand._start(self)

380

381 - def __repr__(self):

382 return "<RemoteShellCommand '%s'>" % repr(self.command)

383

384 -class BuildStep(properties.PropertiesMixin):

385 386 haltOnFailure = False 387 flunkOnWarnings = False 388 flunkOnFailure = False 389 warnOnWarnings = False 390 warnOnFailure = False 391 alwaysRun = False 392 doStepIf = True 393 hideStepIf = False 394 395 # properties set on a build step are, by nature, always runtime properties 396 set_runtime_properties = True 397 398 # 'parms' holds a list of all the parameters we care about, to allow 399 # users to instantiate a subclass of BuildStep with a mixture of 400 # arguments, some of which are for us, some of which are for the subclass 401 # (or a delegate of the subclass, like how ShellCommand delivers many 402 # arguments to the RemoteShellCommand that it creates). Such delegating 403 # subclasses will use this list to figure out which arguments are meant 404 # for us and which should be given to someone else. 405 parms = ['name', 'locks', 406 'haltOnFailure', 407 'flunkOnWarnings', 408 'flunkOnFailure', 409 'warnOnWarnings', 410 'warnOnFailure', 411 'alwaysRun', 412 'progressMetrics', 413 'useProgress', 414 'doStepIf', 415 'hideStepIf', 416 ] 417 418 name = "generic" 419 locks = [] 420 progressMetrics = () # 'time' is implicit 421 useProgress = True # set to False if step is really unpredictable 422 build = None 423 step_status = None 424 progress = None 425

426 - def __init__(self, **kwargs):

427 self.factory = (self.__class__, dict(kwargs)) 428 for p in self.__class__.parms: 429 if kwargs.has_key(p): 430 setattr(self, p, kwargs[p]) 431 del kwargs[p] 432 if kwargs: 433 why = "%s.__init__ got unexpected keyword argument(s) %s" \ 434 % (self, kwargs.keys()) 435 raise TypeError(why) 436 self._pendingLogObservers = [] 437 438 self._acquiringLock = None 439 self.stopped = False

440

441 - def describe(self, done=False):

442 return [self.name]

443

444 - def setBuild(self, build):

445 self.build = build

446

447 - def setBuildSlave(self, buildslave):

448 self.buildslave = buildslave

449

450 - def setDefaultWorkdir(self, workdir):

451 pass

452

453 - def addFactoryArguments(self, **kwargs):

454 self.factory[1].update(kwargs)

455

456 - def getStepFactory(self):

457 return self.factory

458

459 - def setStepStatus(self, step_status):

460 self.step_status = step_status

461

462 - def setupProgress(self):

463 if self.useProgress: 464 sp = progress.StepProgress(self.name, self.progressMetrics) 465 self.progress = sp 466 self.step_status.setProgress(sp) 467 return sp 468 return None

469

470 - def setProgress(self, metric, value):

471 if self.progress: 472 self.progress.setProgress(metric, value)

473

474 - def startStep(self, remote):

475 self.remote = remote 476 self.deferred = defer.Deferred() 477 # convert all locks into their real form 478 lock_list = [] 479 for access in self.locks: 480 if not isinstance(access, locks.LockAccess): 481 # Buildbot 0.7.7 compability: user did not specify access 482 access = access.defaultAccess() 483 lock = self.build.builder.botmaster.getLockByID(access.lockid) 484 lock_list.append((lock, access)) 485 self.locks = lock_list 486 # then narrow SlaveLocks down to the slave that this build is being 487 # run on 488 self.locks = [(l.getLock(self.build.slavebuilder), la) for l, la in self.locks] 489 for l, la in self.locks: 490 if l in self.build.locks: 491 log.msg("Hey, lock %s is claimed by both a Step (%s) and the" 492 " parent Build (%s)" % (l, self, self.build)) 493 raise RuntimeError("lock claimed by both Step and Build") 494 495 # Set the step's text here so that the stepStarted notification sees 496 # the correct description 497 self.step_status.setText(self.describe(False)) 498 self.step_status.stepStarted() 499 500 d = self.acquireLocks() 501 d.addCallback(self._startStep_2) 502 d.addErrback(self.failed) 503 return self.deferred

504

505 - def acquireLocks(self, res=None):

506 self._acquiringLock = None 507 if not self.locks: 508 return defer.succeed(None) 509 if self.stopped: 510 return defer.succeed(None) 511 log.msg("acquireLocks(step %s, locks %s)" % (self, self.locks)) 512 for lock, access in self.locks: 513 if not lock.isAvailable(access): 514 self.step_status.setWaitingForLocks(True) 515 log.msg("step %s waiting for lock %s" % (self, lock)) 516 d = lock.waitUntilMaybeAvailable(self, access) 517 d.addCallback(self.acquireLocks) 518 self._acquiringLock = (lock, access, d) 519 return d 520 # all locks are available, claim them all 521 for lock, access in self.locks: 522 lock.claim(self, access) 523 self.step_status.setWaitingForLocks(False) 524 return defer.succeed(None)

525

526 - def _startStep_2(self, res):

527 if self.stopped: 528 self.finished(EXCEPTION) 529 return 530 531 if self.progress: 532 self.progress.start() 533 534 if isinstance(self.doStepIf, bool): 535 doStep = defer.succeed(self.doStepIf) 536 else: 537 doStep = defer.maybeDeferred(self.doStepIf, self) 538 539 renderables = [] 540 accumulateClassList(self.__class__, 'renderables', renderables) 541 542 for renderable in renderables: 543 setattr(self, renderable, self.build.render(getattr(self, renderable))) 544 545 doStep.addCallback(self._startStep_3) 546 return doStep

547

548 - def _startStep_3(self, doStep):

549 try: 550 if doStep: 551 if self.start() == SKIPPED: 552 doStep = False 553 except: 554 log.msg("BuildStep.startStep exception in .start") 555 self.failed(Failure()) 556 557 if not doStep: 558 self.step_status.setText(self.describe(True) + ['skipped']) 559 self.step_status.setSkipped(True) 560 # this return value from self.start is a shortcut to finishing 561 # the step immediately; we skip calling finished() as 562 # subclasses may have overridden that an expect it to be called 563 # after start() (bug #837) 564 reactor.callLater(0, self._finishFinished, SKIPPED)

565

566 - def start(self):

567 raise NotImplementedError("your subclass must implement this method")

568

569 - def interrupt(self, reason):

570 self.stopped = True 571 if self._acquiringLock: 572 lock, access, d = self._acquiringLock 573 lock.stopWaitingUntilAvailable(self, access, d) 574 d.callback(None)

575

576 - def releaseLocks(self):

577 log.msg("releaseLocks(%s): %s" % (self, self.locks)) 578 for lock, access in self.locks: 579 if lock.isOwner(self, access): 580 lock.release(self, access) 581 else: 582 # This should only happen if we've been interrupted 583 assert self.stopped

584

585 - def finished(self, results):

586 if self.stopped and results != RETRY: 587 # We handle this specially because we don't care about 588 # the return code of an interrupted command; we know 589 # that this should just be exception due to interrupt 590 # At the same time we must respect RETRY status because it's used 591 # to retry interrupted build due to some other issues for example 592 # due to slave lost 593 results = EXCEPTION 594 self.step_status.setText(self.describe(True) + 595 ["interrupted"]) 596 self.step_status.setText2(["interrupted"]) 597 self._finishFinished(results)

598

599 - def _finishFinished(self, results):

600 # internal function to indicate that this step is done; this is separated 601 # from finished() so that subclasses can override finished() 602 if self.progress: 603 self.progress.finish() 604 self.step_status.stepFinished(results) 605 606 hidden = self._maybeEvaluate(self.hideStepIf, results, self) 607 self.step_status.setHidden(hidden) 608 609 self.releaseLocks() 610 self.deferred.callback(results)

611

612 - def failed(self, why):

613 # This can either be a BuildStepFailed exception/failure, meaning we 614 # should call self.finished, or it can be a real exception, which should 615 # be recorded as such. 616 if why.check(BuildStepFailed): 617 self.finished(FAILURE) 618 return 619 620 log.err(why, "BuildStep.failed; traceback follows") 621 try: 622 if self.progress: 623 self.progress.finish() 624 self.addHTMLLog("err.html", formatFailure(why)) 625 self.addCompleteLog("err.text", why.getTraceback()) 626 # could use why.getDetailedTraceback() for more information 627 self.step_status.setText([self.name, "exception"]) 628 self.step_status.setText2([self.name]) 629 self.step_status.stepFinished(EXCEPTION) 630 631 hidden = self._maybeEvaluate(self.hideStepIf, EXCEPTION, self) 632 self.step_status.setHidden(hidden) 633 except: 634 log.msg("exception during failure processing") 635 log.err() 636 # the progress stuff may still be whacked (the StepStatus may 637 # think that it is still running), but the build overall will now 638 # finish 639 try: 640 self.releaseLocks() 641 except: 642 log.msg("exception while releasing locks") 643 log.err() 644 645 log.msg("BuildStep.failed now firing callback") 646 self.deferred.callback(EXCEPTION)

647 648 # utility methods that BuildSteps may find useful 649

650 - def slaveVersion(self, command, oldversion=None):

651 return self.build.getSlaveCommandVersion(command, oldversion)

652

653 - def slaveVersionIsOlderThan(self, command, minversion):

654 sv = self.build.getSlaveCommandVersion(command, None) 655 if sv is None: 656 return True 657 if map(int, sv.split(".")) < map(int, minversion.split(".")): 658 return True 659 return False

660

661 - def getSlaveName(self):

662 return self.build.getSlaveName()

663

664 - def addLog(self, name):

665 loog = self.step_status.addLog(name) 666 self._connectPendingLogObservers() 667 return loog

668

669 - def getLog(self, name):

670 for l in self.step_status.getLogs(): 671 if l.getName() == name: 672 return l 673 raise KeyError("no log named '%s'" % (name,))

674

675 - def addCompleteLog(self, name, text):

676 log.msg("addCompleteLog(%s)" % name) 677 loog = self.step_status.addLog(name) 678 size = loog.chunkSize 679 for start in range(0, len(text), size): 680 loog.addStdout(text[start:start+size]) 681 loog.finish() 682 self._connectPendingLogObservers()

683

684 - def addHTMLLog(self, name, html):

685 log.msg("addHTMLLog(%s)" % name) 686 self.step_status.addHTMLLog(name, html) 687 self._connectPendingLogObservers()

688

689 - def addLogObserver(self, logname, observer):

690 assert interfaces.ILogObserver.providedBy(observer) 691 observer.setStep(self) 692 self._pendingLogObservers.append((logname, observer)) 693 self._connectPendingLogObservers()

694

695 - def _connectPendingLogObservers(self):

696 if not self._pendingLogObservers: 697 return 698 if not self.step_status: 699 return 700 current_logs = {} 701 for loog in self.step_status.getLogs(): 702 current_logs[loog.getName()] = loog 703 for logname, observer in self._pendingLogObservers[:]: 704 if logname in current_logs: 705 observer.setLog(current_logs[logname]) 706 self._pendingLogObservers.remove((logname, observer))

707

708 - def addURL(self, name, url):

709 self.step_status.addURL(name, url)

710

711 - def runCommand(self, c):

712 c.buildslave = self.buildslave 713 d = c.run(self, self.remote) 714 return d

715 716 @staticmethod

717 - def _maybeEvaluate(value, *args, **kwargs):

718 if callable(value): 719 value = value(*args, **kwargs) 720 return value

721 722 components.registerAdapter( 723 lambda step : interfaces.IProperties(step.build), 724 BuildStep, interfaces.IProperties)

725 726 727 -class OutputProgressObserver(LogObserver):

728 length = 0 729

730 - def __init__(self, name):

731 self.name = name

732

733 - def logChunk(self, build, step, log, channel, text):

734 self.length += len(text) 735 self.step.setProgress(self.name, self.length)

736

737 -class LoggingBuildStep(BuildStep):

738 739 progressMetrics = ('output',) 740 logfiles = {} 741 742 parms = BuildStep.parms + ['logfiles', 'lazylogfiles', 'log_eval_func'] 743 cmd = None 744 745 renderables = [ 'logfiles', 'lazylogfiles' ] 746

747 - def __init__(self, logfiles={}, lazylogfiles=False, log_eval_func=None, 748 *args, **kwargs):

749 BuildStep.__init__(self, *args, **kwargs) 750 self.addFactoryArguments(logfiles=logfiles, 751 lazylogfiles=lazylogfiles, 752 log_eval_func=log_eval_func) 753 754 if logfiles and not isinstance(logfiles, dict): 755 config.error( 756 "the ShellCommand 'logfiles' parameter must be a dictionary") 757 758 # merge a class-level 'logfiles' attribute with one passed in as an 759 # argument 760 self.logfiles = self.logfiles.copy() 761 self.logfiles.update(logfiles) 762 self.lazylogfiles = lazylogfiles 763 if log_eval_func and not callable(log_eval_func): 764 config.error( 765 "the 'log_eval_func' paramater must be a callable") 766 self.log_eval_func = log_eval_func 767 self.addLogObserver('stdio', OutputProgressObserver("output"))

768

769 - def addLogFile(self, logname, filename):

770 self.logfiles[logname] = filename

771

772 - def buildCommandKwargs(self):

773 kwargs = dict() 774 kwargs['logfiles'] = self.logfiles 775 return kwargs

776

777 - def startCommand(self, cmd, errorMessages=[]):

778 """ 779 @param cmd: a suitable RemoteCommand which will be launched, with 780 all output being put into our self.stdio_log LogFile 781 """ 782 log.msg("ShellCommand.startCommand(cmd=%s)" % (cmd,)) 783 log.msg(" cmd.args = %r" % (cmd.args)) 784 self.cmd = cmd # so we can interrupt it 785 self.step_status.setText(self.describe(False)) 786 787 # stdio is the first log 788 self.stdio_log = stdio_log = self.addLog("stdio") 789 cmd.useLog(stdio_log, True) 790 for em in errorMessages: 791 stdio_log.addHeader(em) 792 # TODO: consider setting up self.stdio_log earlier, and have the 793 # code that passes in errorMessages instead call 794 # self.stdio_log.addHeader() directly. 795 796 # there might be other logs 797 self.setupLogfiles(cmd, self.logfiles) 798 799 d = self.runCommand(cmd) # might raise ConnectionLost 800 d.addCallback(lambda res: self.commandComplete(cmd)) 801 d.addCallback(lambda res: self.createSummary(cmd.logs['stdio'])) 802 d.addCallback(lambda res: self.evaluateCommand(cmd)) # returns results 803 def _gotResults(results): 804 self.setStatus(cmd, results) 805 return results

806 d.addCallback(_gotResults) # returns results 807 d.addCallbacks(self.finished, self.checkDisconnect) 808 d.addErrback(self.failed)

809

810 - def setupLogfiles(self, cmd, logfiles):

811 for logname,remotefilename in logfiles.items(): 812 if self.lazylogfiles: 813 # Ask RemoteCommand to watch a logfile, but only add 814 # it when/if we see any data. 815 # 816 # The dummy default argument local_logname is a work-around for 817 # Python name binding; default values are bound by value, but 818 # captured variables in the body are bound by name. 819 callback = lambda cmd_arg, local_logname=logname: self.addLog(local_logname) 820 cmd.useLogDelayed(logname, callback, True) 821 else: 822 # tell the BuildStepStatus to add a LogFile 823 newlog = self.addLog(logname) 824 # and tell the RemoteCommand to feed it 825 cmd.useLog(newlog, True)

826

827 - def interrupt(self, reason):

828 # TODO: consider adding an INTERRUPTED or STOPPED status to use 829 # instead of FAILURE, might make the text a bit more clear. 830 # 'reason' can be a Failure, or text 831 BuildStep.interrupt(self, reason) 832 if self.step_status.isWaitingForLocks(): 833 self.addCompleteLog('interrupt while waiting for locks', str(reason)) 834 else: 835 self.addCompleteLog('interrupt', str(reason)) 836 837 if self.cmd: 838 d = self.cmd.interrupt(reason) 839 d.addErrback(log.err, 'while interrupting command')

840

841 - def checkDisconnect(self, f):

842 f.trap(error.ConnectionLost) 843 self.step_status.setText(self.describe(True) + 844 ["exception", "slave", "lost"]) 845 self.step_status.setText2(["exception", "slave", "lost"]) 846 return self.finished(RETRY)

847

848 - def commandComplete(self, cmd):

849 pass

850

851 - def createSummary(self, stdio):

852 pass

853

854 - def evaluateCommand(self, cmd):

855 if self.log_eval_func: 856 return self.log_eval_func(cmd, self.step_status) 857 if cmd.rc != 0: 858 return FAILURE 859 return SUCCESS

860

861 - def getText(self, cmd, results):

862 if results == SUCCESS: 863 return self.describe(True) 864 elif results == WARNINGS: 865 return self.describe(True) + ["warnings"] 866 elif results == EXCEPTION: 867 return self.describe(True) + ["exception"] 868 else: 869 return self.describe(True) + ["failed"]

870

871 - def getText2(self, cmd, results):

872 return [self.name]

873

874 - def maybeGetText2(self, cmd, results):

875 if results == SUCCESS: 876 # successful steps do not add anything to the build's text 877 pass 878 elif results == WARNINGS: 879 if (self.flunkOnWarnings or self.warnOnWarnings): 880 # we're affecting the overall build, so tell them why 881 return self.getText2(cmd, results) 882 else: 883 if (self.haltOnFailure or self.flunkOnFailure 884 or self.warnOnFailure): 885 # we're affecting the overall build, so tell them why 886 return self.getText2(cmd, results) 887 return []

888

889 - def setStatus(self, cmd, results):

890 # this is good enough for most steps, but it can be overridden to 891 # get more control over the displayed text 892 self.step_status.setText(self.getText(cmd, results)) 893 self.step_status.setText2(self.maybeGetText2(cmd, results))

894

895 896 # Parses the logs for a list of regexs. Meant to be invoked like: 897 # regexes = ((re.compile(...), FAILURE), (re.compile(...), WARNINGS)) 898 # self.addStep(ShellCommand, 899 # command=..., 900 # ..., 901 # log_eval_func=lambda c,s: regex_log_evaluator(c, s, regexs) 902 # ) 903 -def regex_log_evaluator(cmd, step_status, regexes):

904 worst = SUCCESS 905 if cmd.rc != 0: 906 worst = FAILURE 907 for err, possible_status in regexes: 908 # worst_status returns the worse of the two status' passed to it. 909 # we won't be changing "worst" unless possible_status is worse than it, 910 # so we don't even need to check the log if that's the case 911 if worst_status(worst, possible_status) == possible_status: 912 if isinstance(err, (basestring)): 913 err = re.compile(".*%s.*" % err, re.DOTALL) 914 for l in cmd.logs.values(): 915 if err.search(l.getText()): 916 worst = possible_status 917 return worst

918 919 # (WithProperties used to be available in this module) 920 from buildbot.process.properties import WithProperties 921 _hush_pyflakes = [WithProperties] 922 del _hush_pyflakes 923

Source Code for Module buildbot.process.buildstep