buildbot.process.buildstep

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 import re 17 18 from zope.interface import implements 19 from twisted.internet import reactor, defer, error 20 from twisted.protocols import basic 21 from twisted.spread import pb 22 from twisted.python import log, components 23 from twisted.python.failure import Failure 24 from twisted.web.util import formatFailure 25 from twisted.python.reflect import accumulateClassList 26 27 from buildbot import interfaces, locks, util, config 28 from buildbot.status import progress 29 from buildbot.status.results import SUCCESS, WARNINGS, FAILURE, SKIPPED, \ 30 EXCEPTION, RETRY, worst_status 31 from buildbot.process import metrics, properties

32 33 -class BuildStepFailed(Exception):

34 pass

35

36 -class RemoteCommand(pb.Referenceable):

37 38 # class-level unique identifier generator for command ids 39 _commandCounter = 0 40 41 active = False 42 rc = None 43 debug = False 44

45 - def __init__(self, remote_command, args, ignore_updates=False, 46 collectStdout=False, decodeRC={0:SUCCESS}):

47 self.logs = {} 48 self.delayedLogs = {} 49 self._closeWhenFinished = {} 50 self.collectStdout = collectStdout 51 self.stdout = '' 52 53 self._startTime = None 54 self._remoteElapsed = None 55 self.remote_command = remote_command 56 self.args = args 57 self.ignore_updates = ignore_updates 58 self.decodeRC = decodeRC

59

60 - def __repr__(self):

61 return "<RemoteCommand '%s' at %d>" % (self.remote_command, id(self))

62

63 - def run(self, step, remote):

64 self.active = True 65 self.step = step 66 self.remote = remote 67 68 # generate a new command id 69 cmd_id = RemoteCommand._commandCounter 70 RemoteCommand._commandCounter += 1 71 self.commandID = "%d" % cmd_id 72 73 log.msg("%s: RemoteCommand.run [%s]" % (self, self.commandID)) 74 self.deferred = defer.Deferred() 75 76 d = defer.maybeDeferred(self._start) 77 78 # _finished is called with an error for unknown commands, errors 79 # that occur while the command is starting (including OSErrors in 80 # exec()), StaleBroker (when the connection was lost before we 81 # started), and pb.PBConnectionLost (when the slave isn't responding 82 # over this connection, perhaps it had a power failure, or NAT 83 # weirdness). If this happens, self.deferred is fired right away. 84 d.addErrback(self._finished) 85 86 # Connections which are lost while the command is running are caught 87 # when our parent Step calls our .lostRemote() method. 88 return self.deferred

89

90 - def useLog(self, log, closeWhenFinished=False, logfileName=None):

91 assert interfaces.ILogFile.providedBy(log) 92 if not logfileName: 93 logfileName = log.getName() 94 assert logfileName not in self.logs 95 assert logfileName not in self.delayedLogs 96 self.logs[logfileName] = log 97 self._closeWhenFinished[logfileName] = closeWhenFinished

98

99 - def useLogDelayed(self, logfileName, activateCallBack, closeWhenFinished=False):

100 assert logfileName not in self.logs 101 assert logfileName not in self.delayedLogs 102 self.delayedLogs[logfileName] = (activateCallBack, closeWhenFinished)

103

104 - def _start(self):

105 self.updates = {} 106 self._startTime = util.now() 107 108 # This method only initiates the remote command. 109 # We will receive remote_update messages as the command runs. 110 # We will get a single remote_complete when it finishes. 111 # We should fire self.deferred when the command is done. 112 d = self.remote.callRemote("startCommand", self, self.commandID, 113 self.remote_command, self.args) 114 return d

115

116 - def _finished(self, failure=None):

117 self.active = False 118 # call .remoteComplete. If it raises an exception, or returns the 119 # Failure that we gave it, our self.deferred will be errbacked. If 120 # it does not (either it ate the Failure or there the step finished 121 # normally and it didn't raise a new exception), self.deferred will 122 # be callbacked. 123 d = defer.maybeDeferred(self.remoteComplete, failure) 124 # arrange for the callback to get this RemoteCommand instance 125 # instead of just None 126 d.addCallback(lambda r: self) 127 # this fires the original deferred we returned from .run(), 128 # with self as the result, or a failure 129 d.addBoth(self.deferred.callback)

130

131 - def interrupt(self, why):

132 log.msg("RemoteCommand.interrupt", self, why) 133 if not self.active: 134 log.msg(" but this RemoteCommand is already inactive") 135 return defer.succeed(None) 136 if not self.remote: 137 log.msg(" but our .remote went away") 138 return defer.succeed(None) 139 if isinstance(why, Failure) and why.check(error.ConnectionLost): 140 log.msg("RemoteCommand.disconnect: lost slave") 141 self.remote = None 142 self._finished(why) 143 return defer.succeed(None) 144 145 # tell the remote command to halt. Returns a Deferred that will fire 146 # when the interrupt command has been delivered. 147 148 d = defer.maybeDeferred(self.remote.callRemote, "interruptCommand", 149 self.commandID, str(why)) 150 # the slave may not have remote_interruptCommand 151 d.addErrback(self._interruptFailed) 152 return d

153

154 - def _interruptFailed(self, why):

155 log.msg("RemoteCommand._interruptFailed", self) 156 # TODO: forcibly stop the Command now, since we can't stop it 157 # cleanly 158 return None

159

160 - def remote_update(self, updates):

161 """ 162 I am called by the slave's L{buildbot.slave.bot.SlaveBuilder} so 163 I can receive updates from the running remote command. 164 165 @type updates: list of [object, int] 166 @param updates: list of updates from the remote command 167 """ 168 self.buildslave.messageReceivedFromSlave() 169 max_updatenum = 0 170 for (update, num) in updates: 171 #log.msg("update[%d]:" % num) 172 try: 173 if self.active and not self.ignore_updates: 174 self.remoteUpdate(update) 175 except: 176 # log failure, terminate build, let slave retire the update 177 self._finished(Failure()) 178 # TODO: what if multiple updates arrive? should 179 # skip the rest but ack them all 180 if num > max_updatenum: 181 max_updatenum = num 182 return max_updatenum

183

184 - def remote_complete(self, failure=None):

185 """ 186 Called by the slave's L{buildbot.slave.bot.SlaveBuilder} to 187 notify me the remote command has finished. 188 189 @type failure: L{twisted.python.failure.Failure} or None 190 191 @rtype: None 192 """ 193 self.buildslave.messageReceivedFromSlave() 194 # call the real remoteComplete a moment later, but first return an 195 # acknowledgement so the slave can retire the completion message. 196 if self.active: 197 reactor.callLater(0, self._finished, failure) 198 return None

199

200 - def addStdout(self, data):

201 if 'stdio' in self.logs: 202 self.logs['stdio'].addStdout(data) 203 if self.collectStdout: 204 self.stdout += data

205

206 - def addStderr(self, data):

207 if 'stdio' in self.logs: 208 self.logs['stdio'].addStderr(data)

209

210 - def addHeader(self, data):

211 if 'stdio' in self.logs: 212 self.logs['stdio'].addHeader(data)

213

214 - def addToLog(self, logname, data):

215 # Activate delayed logs on first data. 216 if logname in self.delayedLogs: 217 (activateCallBack, closeWhenFinished) = self.delayedLogs[logname] 218 del self.delayedLogs[logname] 219 loog = activateCallBack(self) 220 self.logs[logname] = loog 221 self._closeWhenFinished[logname] = closeWhenFinished 222 223 if logname in self.logs: 224 self.logs[logname].addStdout(data) 225 else: 226 log.msg("%s.addToLog: no such log %s" % (self, logname))

227 228 @metrics.countMethod('RemoteCommand.remoteUpdate()')

229 - def remoteUpdate(self, update):

230 if self.debug: 231 for k,v in update.items(): 232 log.msg("Update[%s]: %s" % (k,v)) 233 if update.has_key('stdout'): 234 # 'stdout': data 235 self.addStdout(update['stdout']) 236 if update.has_key('stderr'): 237 # 'stderr': data 238 self.addStderr(update['stderr']) 239 if update.has_key('header'): 240 # 'header': data 241 self.addHeader(update['header']) 242 if update.has_key('log'): 243 # 'log': (logname, data) 244 logname, data = update['log'] 245 self.addToLog(logname, data) 246 if update.has_key('rc'): 247 rc = self.rc = update['rc'] 248 log.msg("%s rc=%s" % (self, rc)) 249 self.addHeader("program finished with exit code %d\n" % rc) 250 if update.has_key('elapsed'): 251 self._remoteElapsed = update['elapsed'] 252 253 # TODO: these should be handled at the RemoteCommand level 254 for k in update: 255 if k not in ('stdout', 'stderr', 'header', 'rc'): 256 if k not in self.updates: 257 self.updates[k] = [] 258 self.updates[k].append(update[k])

259

260 - def remoteComplete(self, maybeFailure):

261 if self._startTime and self._remoteElapsed: 262 delta = (util.now() - self._startTime) - self._remoteElapsed 263 metrics.MetricTimeEvent.log("RemoteCommand.overhead", delta) 264 265 for name,loog in self.logs.items(): 266 if self._closeWhenFinished[name]: 267 if maybeFailure: 268 loog.addHeader("\nremoteFailed: %s" % maybeFailure) 269 else: 270 log.msg("closing log %s" % loog) 271 loog.finish() 272 return maybeFailure

273

274 - def results(self):

275 if self.rc in self.decodeRC: 276 return self.decodeRC[self.rc] 277 return FAILURE

278

279 - def didFail(self):

280 return self.results() == FAILURE

281 LoggedRemoteCommand = RemoteCommand

282 283 284 -class LogObserver:

285 implements(interfaces.ILogObserver) 286

287 - def setStep(self, step):

288 self.step = step

289

290 - def setLog(self, loog):

291 assert interfaces.IStatusLog.providedBy(loog) 292 loog.subscribe(self, True)

293

294 - def logChunk(self, build, step, log, channel, text):

295 if channel == interfaces.LOG_CHANNEL_STDOUT: 296 self.outReceived(text) 297 elif channel == interfaces.LOG_CHANNEL_STDERR: 298 self.errReceived(text)

299 300 # TODO: add a logEnded method? er, stepFinished? 301

302 - def outReceived(self, data):

303 """This will be called with chunks of stdout data. Override this in 304 your observer.""" 305 pass

306

307 - def errReceived(self, data):

308 """This will be called with chunks of stderr data. Override this in 309 your observer.""" 310 pass

311

312 313 -class LogLineObserver(LogObserver):

314 - def __init__(self):

315 self.stdoutParser = basic.LineOnlyReceiver() 316 self.stdoutParser.delimiter = "\n" 317 self.stdoutParser.lineReceived = self.outLineReceived 318 self.stdoutParser.transport = self # for the .disconnecting attribute 319 self.disconnecting = False 320 321 self.stderrParser = basic.LineOnlyReceiver() 322 self.stderrParser.delimiter = "\n" 323 self.stderrParser.lineReceived = self.errLineReceived 324 self.stderrParser.transport = self

325

326 - def setMaxLineLength(self, max_length):

327 """ 328 Set the maximum line length: lines longer than max_length are 329 dropped. Default is 16384 bytes. Use sys.maxint for effective 330 infinity. 331 """ 332 self.stdoutParser.MAX_LENGTH = max_length 333 self.stderrParser.MAX_LENGTH = max_length

334

335 - def outReceived(self, data):

336 self.stdoutParser.dataReceived(data)

337

338 - def errReceived(self, data):

339 self.stderrParser.dataReceived(data)

340

341 - def outLineReceived(self, line):

342 """This will be called with complete stdout lines (not including the 343 delimiter). Override this in your observer.""" 344 pass

345

346 - def errLineReceived(self, line):

347 """This will be called with complete lines of stderr (not including 348 the delimiter). Override this in your observer.""" 349 pass

350

351 352 -class RemoteShellCommand(RemoteCommand):

353 - def __init__(self, workdir, command, env=None, 354 want_stdout=1, want_stderr=1, 355 timeout=20*60, maxTime=None, logfiles={}, 356 usePTY="slave-config", logEnviron=True, 357 collectStdout=False, interruptSignal=None, 358 initialStdin=None, decodeRC={0:SUCCESS}):

359 360 self.command = command # stash .command, set it later 361 if env is not None: 362 # avoid mutating the original master.cfg dictionary. Each 363 # ShellCommand gets its own copy, any start() methods won't be 364 # able to modify the original. 365 env = env.copy() 366 args = {'workdir': workdir, 367 'env': env, 368 'want_stdout': want_stdout, 369 'want_stderr': want_stderr, 370 'logfiles': logfiles, 371 'timeout': timeout, 372 'maxTime': maxTime, 373 'usePTY': usePTY, 374 'logEnviron': logEnviron, 375 'initial_stdin': initialStdin 376 } 377 if interruptSignal is not None: 378 args['interruptSignal'] = interruptSignal 379 RemoteCommand.__init__(self, "shell", args, collectStdout=collectStdout, 380 decodeRC=decodeRC)

381

382 - def _start(self):

383 self.args['command'] = self.command 384 if self.remote_command == "shell": 385 # non-ShellCommand slavecommands are responsible for doing this 386 # fixup themselves 387 if self.step.slaveVersion("shell", "old") == "old": 388 self.args['dir'] = self.args['workdir'] 389 what = "command '%s' in dir '%s'" % (self.args['command'], 390 self.args['workdir']) 391 log.msg(what) 392 return RemoteCommand._start(self)

393

394 - def __repr__(self):

395 return "<RemoteShellCommand '%s'>" % repr(self.command)

396

397 -class _BuildStepFactory(util.ComparableMixin):

398 """ 399 This is a wrapper to record the arguments passed to as BuildStep subclass. 400 We use an instance of this class, rather than a closure mostly to make it 401 easier to test that the right factories are getting created. 402 """ 403 compare_attrs = ['factory', 'args', 'kwargs' ] 404 implements(interfaces.IBuildStepFactory) 405

406 - def __init__(self, factory, *args, **kwargs):

407 self.factory = factory 408 self.args = args 409 self.kwargs = kwargs

410

411 - def buildStep(self):

412 try: 413 return self.factory(*self.args, **self.kwargs) 414 except: 415 log.msg("error while creating step, factory=%s, args=%s, kwargs=%s" 416 % (self.factory, self.args, self.kwargs)) 417 raise

418

419 -class BuildStep(object, properties.PropertiesMixin):

420 421 haltOnFailure = False 422 flunkOnWarnings = False 423 flunkOnFailure = False 424 warnOnWarnings = False 425 warnOnFailure = False 426 alwaysRun = False 427 doStepIf = True 428 hideStepIf = False 429 430 # properties set on a build step are, by nature, always runtime properties 431 set_runtime_properties = True 432 433 # 'parms' holds a list of all the parameters we care about, to allow 434 # users to instantiate a subclass of BuildStep with a mixture of 435 # arguments, some of which are for us, some of which are for the subclass 436 # (or a delegate of the subclass, like how ShellCommand delivers many 437 # arguments to the RemoteShellCommand that it creates). Such delegating 438 # subclasses will use this list to figure out which arguments are meant 439 # for us and which should be given to someone else. 440 parms = ['name', 'locks', 441 'haltOnFailure', 442 'flunkOnWarnings', 443 'flunkOnFailure', 444 'warnOnWarnings', 445 'warnOnFailure', 446 'alwaysRun', 447 'progressMetrics', 448 'useProgress', 449 'doStepIf', 450 'hideStepIf', 451 ] 452 453 name = "generic" 454 locks = [] 455 progressMetrics = () # 'time' is implicit 456 useProgress = True # set to False if step is really unpredictable 457 build = None 458 step_status = None 459 progress = None 460

461 - def __init__(self, **kwargs):

462 for p in self.__class__.parms: 463 if kwargs.has_key(p): 464 setattr(self, p, kwargs[p]) 465 del kwargs[p] 466 if kwargs: 467 why = "%s.__init__ got unexpected keyword argument(s) %s" \ 468 % (self, kwargs.keys()) 469 raise TypeError(why) 470 self._pendingLogObservers = [] 471 472 self._acquiringLock = None 473 self.stopped = False

474

475 - def __new__(klass, *args, **kwargs):

476 self = object.__new__(klass) 477 self._factory = _BuildStepFactory(klass, *args, **kwargs) 478 return self

479

480 - def describe(self, done=False):

481 return [self.name]

482

483 - def setBuild(self, build):

484 self.build = build

485

486 - def setBuildSlave(self, buildslave):

487 self.buildslave = buildslave

488

489 - def setDefaultWorkdir(self, workdir):

490 pass

491

492 - def addFactoryArguments(self, **kwargs):

493 # this is here for backwards compatability 494 pass

495

496 - def _getStepFactory(self):

497 return self._factory

498

499 - def setStepStatus(self, step_status):

500 self.step_status = step_status

501

502 - def setupProgress(self):

503 if self.useProgress: 504 sp = progress.StepProgress(self.name, self.progressMetrics) 505 self.progress = sp 506 self.step_status.setProgress(sp) 507 return sp 508 return None

509

510 - def setProgress(self, metric, value):

511 if self.progress: 512 self.progress.setProgress(metric, value)

513

514 - def startStep(self, remote):

515 self.remote = remote 516 self.deferred = defer.Deferred() 517 # convert all locks into their real form 518 lock_list = [] 519 for access in self.locks: 520 if not isinstance(access, locks.LockAccess): 521 # Buildbot 0.7.7 compability: user did not specify access 522 access = access.defaultAccess() 523 lock = self.build.builder.botmaster.getLockByID(access.lockid) 524 lock_list.append((lock, access)) 525 self.locks = lock_list 526 # then narrow SlaveLocks down to the slave that this build is being 527 # run on 528 self.locks = [(l.getLock(self.build.slavebuilder.slave), la) for l, la in self.locks] 529 for l, la in self.locks: 530 if l in self.build.locks: 531 log.msg("Hey, lock %s is claimed by both a Step (%s) and the" 532 " parent Build (%s)" % (l, self, self.build)) 533 raise RuntimeError("lock claimed by both Step and Build") 534 535 # Set the step's text here so that the stepStarted notification sees 536 # the correct description 537 self.step_status.setText(self.describe(False)) 538 self.step_status.stepStarted() 539 540 d = self.acquireLocks() 541 d.addCallback(self._startStep_2) 542 d.addErrback(self.failed) 543 return self.deferred

544

545 - def acquireLocks(self, res=None):

546 self._acquiringLock = None 547 if not self.locks: 548 return defer.succeed(None) 549 if self.stopped: 550 return defer.succeed(None) 551 log.msg("acquireLocks(step %s, locks %s)" % (self, self.locks)) 552 for lock, access in self.locks: 553 if not lock.isAvailable(self, access): 554 self.step_status.setWaitingForLocks(True) 555 log.msg("step %s waiting for lock %s" % (self, lock)) 556 d = lock.waitUntilMaybeAvailable(self, access) 557 d.addCallback(self.acquireLocks) 558 self._acquiringLock = (lock, access, d) 559 return d 560 # all locks are available, claim them all 561 for lock, access in self.locks: 562 lock.claim(self, access) 563 self.step_status.setWaitingForLocks(False) 564 return defer.succeed(None)

565

566 - def _startStep_2(self, res):

567 if self.stopped: 568 self.finished(EXCEPTION) 569 return 570 571 if self.progress: 572 self.progress.start() 573 574 if isinstance(self.doStepIf, bool): 575 doStep = defer.succeed(self.doStepIf) 576 else: 577 doStep = defer.maybeDeferred(self.doStepIf, self) 578 579 renderables = [] 580 accumulateClassList(self.__class__, 'renderables', renderables) 581 582 def setRenderable(res, attr): 583 setattr(self, attr, res)

584 585 dl = [ doStep ] 586 for renderable in renderables: 587 d = self.build.render(getattr(self, renderable)) 588 d.addCallback(setRenderable, renderable) 589 dl.append(d) 590 dl = defer.gatherResults(dl) 591 592 dl.addCallback(self._startStep_3) 593 return dl

594 595 @defer.inlineCallbacks

596 - def _startStep_3(self, doStep):

597 doStep = doStep[0] 598 try: 599 if doStep: 600 result = yield defer.maybeDeferred(self.start) 601 if result == SKIPPED: 602 doStep = False 603 except: 604 log.msg("BuildStep.startStep exception in .start") 605 self.failed(Failure()) 606 607 if not doStep: 608 self.step_status.setText(self.describe(True) + ['skipped']) 609 self.step_status.setSkipped(True) 610 # this return value from self.start is a shortcut to finishing 611 # the step immediately; we skip calling finished() as 612 # subclasses may have overridden that an expect it to be called 613 # after start() (bug #837) 614 reactor.callLater(0, self._finishFinished, SKIPPED)

615

616 - def start(self):

617 raise NotImplementedError("your subclass must implement this method")

618

619 - def interrupt(self, reason):

620 self.stopped = True 621 if self._acquiringLock: 622 lock, access, d = self._acquiringLock 623 lock.stopWaitingUntilAvailable(self, access, d) 624 d.callback(None)

625

626 - def releaseLocks(self):

627 log.msg("releaseLocks(%s): %s" % (self, self.locks)) 628 for lock, access in self.locks: 629 if lock.isOwner(self, access): 630 lock.release(self, access) 631 else: 632 # This should only happen if we've been interrupted 633 assert self.stopped

634

635 - def finished(self, results):

636 if self.stopped and results != RETRY: 637 # We handle this specially because we don't care about 638 # the return code of an interrupted command; we know 639 # that this should just be exception due to interrupt 640 # At the same time we must respect RETRY status because it's used 641 # to retry interrupted build due to some other issues for example 642 # due to slave lost 643 results = EXCEPTION 644 self.step_status.setText(self.describe(True) + 645 ["interrupted"]) 646 self.step_status.setText2(["interrupted"]) 647 self._finishFinished(results)

648

649 - def _finishFinished(self, results):

650 # internal function to indicate that this step is done; this is separated 651 # from finished() so that subclasses can override finished() 652 if self.progress: 653 self.progress.finish() 654 655 try: 656 hidden = self._maybeEvaluate(self.hideStepIf, results, self) 657 except Exception: 658 why = Failure() 659 self.addHTMLLog("err.html", formatFailure(why)) 660 self.addCompleteLog("err.text", why.getTraceback()) 661 results = EXCEPTION 662 hidden = False 663 664 self.step_status.stepFinished(results) 665 self.step_status.setHidden(hidden) 666 667 self.releaseLocks() 668 self.deferred.callback(results)

669

670 - def failed(self, why):

671 # This can either be a BuildStepFailed exception/failure, meaning we 672 # should call self.finished, or it can be a real exception, which should 673 # be recorded as such. 674 if why.check(BuildStepFailed): 675 self.finished(FAILURE) 676 return 677 678 log.err(why, "BuildStep.failed; traceback follows") 679 try: 680 if self.progress: 681 self.progress.finish() 682 self.addHTMLLog("err.html", formatFailure(why)) 683 self.addCompleteLog("err.text", why.getTraceback()) 684 # could use why.getDetailedTraceback() for more information 685 self.step_status.setText([self.name, "exception"]) 686 self.step_status.setText2([self.name]) 687 self.step_status.stepFinished(EXCEPTION) 688 689 hidden = self._maybeEvaluate(self.hideStepIf, EXCEPTION, self) 690 self.step_status.setHidden(hidden) 691 except: 692 log.msg("exception during failure processing") 693 log.err() 694 # the progress stuff may still be whacked (the StepStatus may 695 # think that it is still running), but the build overall will now 696 # finish 697 try: 698 self.releaseLocks() 699 except: 700 log.msg("exception while releasing locks") 701 log.err() 702 703 log.msg("BuildStep.failed now firing callback") 704 self.deferred.callback(EXCEPTION)

705 706 # utility methods that BuildSteps may find useful 707

708 - def slaveVersion(self, command, oldversion=None):

709 return self.build.getSlaveCommandVersion(command, oldversion)

710

711 - def slaveVersionIsOlderThan(self, command, minversion):

712 sv = self.build.getSlaveCommandVersion(command, None) 713 if sv is None: 714 return True 715 if map(int, sv.split(".")) < map(int, minversion.split(".")): 716 return True 717 return False

718

719 - def getSlaveName(self):

720 return self.build.getSlaveName()

721

722 - def addLog(self, name):

723 loog = self.step_status.addLog(name) 724 self._connectPendingLogObservers() 725 return loog

726

727 - def getLog(self, name):

728 for l in self.step_status.getLogs(): 729 if l.getName() == name: 730 return l 731 raise KeyError("no log named '%s'" % (name,))

732

733 - def addCompleteLog(self, name, text):

734 log.msg("addCompleteLog(%s)" % name) 735 loog = self.step_status.addLog(name) 736 size = loog.chunkSize 737 for start in range(0, len(text), size): 738 loog.addStdout(text[start:start+size]) 739 loog.finish() 740 self._connectPendingLogObservers()

741

742 - def addHTMLLog(self, name, html):

743 log.msg("addHTMLLog(%s)" % name) 744 self.step_status.addHTMLLog(name, html) 745 self._connectPendingLogObservers()

746

747 - def addLogObserver(self, logname, observer):

748 assert interfaces.ILogObserver.providedBy(observer) 749 observer.setStep(self) 750 self._pendingLogObservers.append((logname, observer)) 751 self._connectPendingLogObservers()

752

753 - def _connectPendingLogObservers(self):

754 if not self._pendingLogObservers: 755 return 756 if not self.step_status: 757 return 758 current_logs = {} 759 for loog in self.step_status.getLogs(): 760 current_logs[loog.getName()] = loog 761 for logname, observer in self._pendingLogObservers[:]: 762 if logname in current_logs: 763 observer.setLog(current_logs[logname]) 764 self._pendingLogObservers.remove((logname, observer))

765

766 - def addURL(self, name, url):

767 self.step_status.addURL(name, url)

768

769 - def runCommand(self, c):

770 c.buildslave = self.buildslave 771 d = c.run(self, self.remote) 772 return d

773 774 @staticmethod

775 - def _maybeEvaluate(value, *args, **kwargs):

776 if callable(value): 777 value = value(*args, **kwargs) 778 return value

779 780 components.registerAdapter( 781 BuildStep._getStepFactory, 782 BuildStep, interfaces.IBuildStepFactory) 783 components.registerAdapter( 784 lambda step : interfaces.IProperties(step.build), 785 BuildStep, interfaces.IProperties)

786 787 788 -class OutputProgressObserver(LogObserver):

789 length = 0 790

791 - def __init__(self, name):

792 self.name = name

793

794 - def logChunk(self, build, step, log, channel, text):

795 self.length += len(text) 796 self.step.setProgress(self.name, self.length)

797

798 -class LoggingBuildStep(BuildStep):

799 800 progressMetrics = ('output',) 801 logfiles = {} 802 803 parms = BuildStep.parms + ['logfiles', 'lazylogfiles', 'log_eval_func'] 804 cmd = None 805 806 renderables = [ 'logfiles', 'lazylogfiles' ] 807

808 - def __init__(self, logfiles={}, lazylogfiles=False, log_eval_func=None, 809 *args, **kwargs):

810 BuildStep.__init__(self, *args, **kwargs) 811 812 if logfiles and not isinstance(logfiles, dict): 813 config.error( 814 "the ShellCommand 'logfiles' parameter must be a dictionary") 815 816 # merge a class-level 'logfiles' attribute with one passed in as an 817 # argument 818 self.logfiles = self.logfiles.copy() 819 self.logfiles.update(logfiles) 820 self.lazylogfiles = lazylogfiles 821 if log_eval_func and not callable(log_eval_func): 822 config.error( 823 "the 'log_eval_func' paramater must be a callable") 824 self.log_eval_func = log_eval_func 825 self.addLogObserver('stdio', OutputProgressObserver("output"))

826

827 - def addLogFile(self, logname, filename):

828 self.logfiles[logname] = filename

829

830 - def buildCommandKwargs(self):

831 kwargs = dict() 832 kwargs['logfiles'] = self.logfiles 833 return kwargs

834

835 - def startCommand(self, cmd, errorMessages=[]):

836 """ 837 @param cmd: a suitable RemoteCommand which will be launched, with 838 all output being put into our self.stdio_log LogFile 839 """ 840 log.msg("ShellCommand.startCommand(cmd=%s)" % (cmd,)) 841 log.msg(" cmd.args = %r" % (cmd.args)) 842 self.cmd = cmd # so we can interrupt it 843 self.step_status.setText(self.describe(False)) 844 845 # stdio is the first log 846 self.stdio_log = stdio_log = self.addLog("stdio") 847 cmd.useLog(stdio_log, True) 848 for em in errorMessages: 849 stdio_log.addHeader(em) 850 # TODO: consider setting up self.stdio_log earlier, and have the 851 # code that passes in errorMessages instead call 852 # self.stdio_log.addHeader() directly. 853 854 # there might be other logs 855 self.setupLogfiles(cmd, self.logfiles) 856 857 d = self.runCommand(cmd) # might raise ConnectionLost 858 d.addCallback(lambda res: self.commandComplete(cmd)) 859 d.addCallback(lambda res: self.createSummary(cmd.logs['stdio'])) 860 d.addCallback(lambda res: self.evaluateCommand(cmd)) # returns results 861 def _gotResults(results): 862 self.setStatus(cmd, results) 863 return results

864 d.addCallback(_gotResults) # returns results 865 d.addCallbacks(self.finished, self.checkDisconnect) 866 d.addErrback(self.failed)

867

868 - def setupLogfiles(self, cmd, logfiles):

869 for logname,remotefilename in logfiles.items(): 870 if self.lazylogfiles: 871 # Ask RemoteCommand to watch a logfile, but only add 872 # it when/if we see any data. 873 # 874 # The dummy default argument local_logname is a work-around for 875 # Python name binding; default values are bound by value, but 876 # captured variables in the body are bound by name. 877 callback = lambda cmd_arg, local_logname=logname: self.addLog(local_logname) 878 cmd.useLogDelayed(logname, callback, True) 879 else: 880 # tell the BuildStepStatus to add a LogFile 881 newlog = self.addLog(logname) 882 # and tell the RemoteCommand to feed it 883 cmd.useLog(newlog, True)

884

885 - def interrupt(self, reason):

886 # TODO: consider adding an INTERRUPTED or STOPPED status to use 887 # instead of FAILURE, might make the text a bit more clear. 888 # 'reason' can be a Failure, or text 889 BuildStep.interrupt(self, reason) 890 if self.step_status.isWaitingForLocks(): 891 self.addCompleteLog('interrupt while waiting for locks', str(reason)) 892 else: 893 self.addCompleteLog('interrupt', str(reason)) 894 895 if self.cmd: 896 d = self.cmd.interrupt(reason) 897 d.addErrback(log.err, 'while interrupting command')

898

899 - def checkDisconnect(self, f):

900 f.trap(error.ConnectionLost) 901 self.step_status.setText(self.describe(True) + 902 ["exception", "slave", "lost"]) 903 self.step_status.setText2(["exception", "slave", "lost"]) 904 return self.finished(RETRY)

905

906 - def commandComplete(self, cmd):

907 pass

908

909 - def createSummary(self, stdio):

910 pass

911

912 - def evaluateCommand(self, cmd):

913 if self.log_eval_func: 914 return self.log_eval_func(cmd, self.step_status) 915 return cmd.results()

916

917 - def getText(self, cmd, results):

918 if results == SUCCESS: 919 return self.describe(True) 920 elif results == WARNINGS: 921 return self.describe(True) + ["warnings"] 922 elif results == EXCEPTION: 923 return self.describe(True) + ["exception"] 924 else: 925 return self.describe(True) + ["failed"]

926

927 - def getText2(self, cmd, results):

928 return [self.name]

929

930 - def maybeGetText2(self, cmd, results):

931 if results == SUCCESS: 932 # successful steps do not add anything to the build's text 933 pass 934 elif results == WARNINGS: 935 if (self.flunkOnWarnings or self.warnOnWarnings): 936 # we're affecting the overall build, so tell them why 937 return self.getText2(cmd, results) 938 else: 939 if (self.haltOnFailure or self.flunkOnFailure 940 or self.warnOnFailure): 941 # we're affecting the overall build, so tell them why 942 return self.getText2(cmd, results) 943 return []

944

945 - def setStatus(self, cmd, results):

946 # this is good enough for most steps, but it can be overridden to 947 # get more control over the displayed text 948 self.step_status.setText(self.getText(cmd, results)) 949 self.step_status.setText2(self.maybeGetText2(cmd, results))

950

951 952 # Parses the logs for a list of regexs. Meant to be invoked like: 953 # regexes = ((re.compile(...), FAILURE), (re.compile(...), WARNINGS)) 954 # self.addStep(ShellCommand, 955 # command=..., 956 # ..., 957 # log_eval_func=lambda c,s: regex_log_evaluator(c, s, regexs) 958 # ) 959 -def regex_log_evaluator(cmd, step_status, regexes):

960 worst = cmd.results() 961 for err, possible_status in regexes: 962 # worst_status returns the worse of the two status' passed to it. 963 # we won't be changing "worst" unless possible_status is worse than it, 964 # so we don't even need to check the log if that's the case 965 if worst_status(worst, possible_status) == possible_status: 966 if isinstance(err, (basestring)): 967 err = re.compile(".*%s.*" % err, re.DOTALL) 968 for l in cmd.logs.values(): 969 if err.search(l.getText()): 970 worst = possible_status 971 return worst

972 973 # (WithProperties used to be available in this module) 974 from buildbot.process.properties import WithProperties 975 _hush_pyflakes = [WithProperties] 976 del _hush_pyflakes 977

Source Code for Module buildbot.process.buildstep