buildbot.buildslave

1 # Portions copyright Canonical Ltd. 2009 2 3 import time 4 from email.Message import Message 5 from email.Utils import formatdate 6 from zope.interface import implements 7 from twisted.python import log 8 from twisted.internet import defer, reactor 9 from twisted.application import service 10 import twisted.spread.pb 11 12 from buildbot.pbutil import NewCredPerspective 13 from buildbot.status.builder import SlaveStatus 14 from buildbot.status.mail import MailNotifier 15 from buildbot.interfaces import IBuildSlave, ILatentBuildSlave 16 from buildbot.process.properties import Properties 17 18 import sys 19 if sys.version_info[:3] < (2,4,0): 20 from sets import Set as set 21

22 -class AbstractBuildSlave(NewCredPerspective, service.MultiService):

23 """This is the master-side representative for a remote buildbot slave. 24 There is exactly one for each slave described in the config file (the 25 c['slaves'] list). When buildbots connect in (.attach), they get a 26 reference to this instance. The BotMaster object is stashed as the 27 .botmaster attribute. The BotMaster is also our '.parent' Service. 28 29 I represent a build slave -- a remote machine capable of 30 running builds. I am instantiated by the configuration file, and can be 31 subclassed to add extra functionality.""" 32 33 implements(IBuildSlave) 34

35 - def __init__(self, name, password, max_builds=None, 36 notify_on_missing=[], missing_timeout=3600, 37 properties={}):

38 """ 39 @param name: botname this machine will supply when it connects 40 @param password: password this machine will supply when 41 it connects 42 @param max_builds: maximum number of simultaneous builds that will 43 be run concurrently on this buildslave (the 44 default is None for no limit) 45 @param properties: properties that will be applied to builds run on 46 this slave 47 @type properties: dictionary 48 """ 49 service.MultiService.__init__(self) 50 self.slavename = name 51 self.password = password 52 self.botmaster = None # no buildmaster yet 53 self.slave_status = SlaveStatus(name) 54 self.slave = None # a RemoteReference to the Bot, when connected 55 self.slave_commands = None 56 self.slavebuilders = {} 57 self.max_builds = max_builds 58 59 self.properties = Properties() 60 self.properties.update(properties, "BuildSlave") 61 self.properties.setProperty("slavename", name, "BuildSlave") 62 63 self.lastMessageReceived = 0 64 if isinstance(notify_on_missing, str): 65 notify_on_missing = [notify_on_missing] 66 self.notify_on_missing = notify_on_missing 67 for i in notify_on_missing: 68 assert isinstance(i, str) 69 self.missing_timeout = missing_timeout 70 self.missing_timer = None

71

72 - def update(self, new):

73 """ 74 Given a new BuildSlave, configure this one identically. Because 75 BuildSlave objects are remotely referenced, we can't replace them 76 without disconnecting the slave, yet there's no reason to do that. 77 """ 78 # the reconfiguration logic should guarantee this: 79 assert self.slavename == new.slavename 80 assert self.password == new.password 81 assert self.__class__ == new.__class__ 82 self.max_builds = new.max_builds

83

84 - def __repr__(self):

85 if self.botmaster: 86 builders = self.botmaster.getBuildersForSlave(self.slavename) 87 return "<%s '%s', current builders: %s>" % \ 88 (self.__class__.__name__, self.slavename, 89 ','.join(map(lambda b: b.name, builders))) 90 else: 91 return "<%s '%s', (no builders yet)>" % \ 92 (self.__class__.__name__, self.slavename)

93

94 - def setBotmaster(self, botmaster):

95 assert not self.botmaster, "BuildSlave already has a botmaster" 96 self.botmaster = botmaster 97 self.startMissingTimer()

98

99 - def stopMissingTimer(self):

100 if self.missing_timer: 101 self.missing_timer.cancel() 102 self.missing_timer = None

103

104 - def startMissingTimer(self):

105 if self.notify_on_missing and self.missing_timeout and self.parent: 106 self.stopMissingTimer() # in case it's already running 107 self.missing_timer = reactor.callLater(self.missing_timeout, 108 self._missing_timer_fired)

109

110 - def _missing_timer_fired(self):

111 self.missing_timer = None 112 # notify people, but only if we're still in the config 113 if not self.parent: 114 return 115 116 buildmaster = self.botmaster.parent 117 status = buildmaster.getStatus() 118 text = "The Buildbot working for '%s'\n" % status.getProjectName() 119 text += ("has noticed that the buildslave named %s went away\n" % 120 self.slavename) 121 text += "\n" 122 text += ("It last disconnected at %s (buildmaster-local time)\n" % 123 time.ctime(time.time() - self.missing_timeout)) # approx 124 text += "\n" 125 text += "The admin on record (as reported by BUILDSLAVE:info/admin)\n" 126 text += "was '%s'.\n" % self.slave_status.getAdmin() 127 text += "\n" 128 text += "Sincerely,\n" 129 text += " The Buildbot\n" 130 text += " %s\n" % status.getProjectURL() 131 subject = "Buildbot: buildslave %s was lost" % self.slavename 132 return self._mail_missing_message(subject, text)

133 134

135 - def updateSlave(self):

136 """Called to add or remove builders after the slave has connected. 137 138 @return: a Deferred that indicates when an attached slave has 139 accepted the new builders and/or released the old ones.""" 140 if self.slave: 141 return self.sendBuilderList() 142 else: 143 return defer.succeed(None)

144

145 - def updateSlaveStatus(self, buildStarted=None, buildFinished=None):

146 if buildStarted: 147 self.slave_status.buildStarted(buildStarted) 148 if buildFinished: 149 self.slave_status.buildFinished(buildFinished)

150

151 - def attached(self, bot):

152 """This is called when the slave connects. 153 154 @return: a Deferred that fires with a suitable pb.IPerspective to 155 give to the slave (i.e. 'self')""" 156 157 if self.slave: 158 # uh-oh, we've got a duplicate slave. The most likely 159 # explanation is that the slave is behind a slow link, thinks we 160 # went away, and has attempted to reconnect, so we've got two 161 # "connections" from the same slave, but the previous one is 162 # stale. Give the new one precedence. 163 log.msg("duplicate slave %s replacing old one" % self.slavename) 164 165 # just in case we've got two identically-configured slaves, 166 # report the IP addresses of both so someone can resolve the 167 # squabble 168 tport = self.slave.broker.transport 169 log.msg("old slave was connected from", tport.getPeer()) 170 log.msg("new slave is from", bot.broker.transport.getPeer()) 171 d = self.disconnect() 172 else: 173 d = defer.succeed(None) 174 # now we go through a sequence of calls, gathering information, then 175 # tell the Botmaster that it can finally give this slave to all the 176 # Builders that care about it. 177 178 # we accumulate slave information in this 'state' dictionary, then 179 # set it atomically if we make it far enough through the process 180 state = {} 181 182 # Reset graceful shutdown status 183 self.slave_status.setGraceful(False) 184 # We want to know when the graceful shutdown flag changes 185 self.slave_status.addGracefulWatcher(self._gracefulChanged) 186 187 def _log_attachment_on_slave(res): 188 d1 = bot.callRemote("print", "attached") 189 d1.addErrback(lambda why: None) 190 return d1

191 d.addCallback(_log_attachment_on_slave) 192 193 def _get_info(res): 194 d1 = bot.callRemote("getSlaveInfo") 195 def _got_info(info): 196 log.msg("Got slaveinfo from '%s'" % self.slavename) 197 # TODO: info{} might have other keys 198 state["admin"] = info.get("admin") 199 state["host"] = info.get("host") 200 state["access_uri"] = info.get("access_uri", None)

201 def _info_unavailable(why): 202 # maybe an old slave, doesn't implement remote_getSlaveInfo 203 log.msg("BuildSlave.info_unavailable") 204 log.err(why) 205 d1.addCallbacks(_got_info, _info_unavailable) 206 return d1 207 d.addCallback(_get_info) 208 209 def _get_version(res): 210 d1 = bot.callRemote("getVersion") 211 def _got_version(version): 212 state["version"] = version 213 def _version_unavailable(why): 214 # probably an old slave 215 log.msg("BuildSlave.version_unavailable") 216 log.err(why) 217 d1.addCallbacks(_got_version, _version_unavailable) 218 d.addCallback(_get_version) 219 220 def _get_commands(res): 221 d1 = bot.callRemote("getCommands") 222 def _got_commands(commands): 223 state["slave_commands"] = commands 224 def _commands_unavailable(why): 225 # probably an old slave 226 log.msg("BuildSlave._commands_unavailable") 227 if why.check(AttributeError): 228 return 229 log.err(why) 230 d1.addCallbacks(_got_commands, _commands_unavailable) 231 return d1 232 d.addCallback(_get_commands) 233 234 def _accept_slave(res): 235 self.slave_status.setAdmin(state.get("admin")) 236 self.slave_status.setHost(state.get("host")) 237 self.slave_status.setAccessURI(state.get("access_uri")) 238 self.slave_status.setVersion(state.get("version")) 239 self.slave_status.setConnected(True) 240 self.slave_commands = state.get("slave_commands") 241 self.slave = bot 242 log.msg("bot attached") 243 self.messageReceivedFromSlave() 244 self.stopMissingTimer() 245 self.botmaster.parent.status.slaveConnected(self.slavename) 246 247 return self.updateSlave() 248 d.addCallback(_accept_slave) 249 d.addCallback(lambda res: self.botmaster.triggerNewBuildCheck()) 250 251 # Finally, the slave gets a reference to this BuildSlave. They 252 # receive this later, after we've started using them. 253 d.addCallback(lambda res: self) 254 return d 255

256 - def messageReceivedFromSlave(self):

257 now = time.time() 258 self.lastMessageReceived = now 259 self.slave_status.setLastMessageReceived(now)

260

261 - def detached(self, mind):

262 self.slave = None 263 self.slave_status.removeGracefulWatcher(self._gracefulChanged) 264 self.slave_status.setConnected(False) 265 log.msg("BuildSlave.detached(%s)" % self.slavename) 266 self.botmaster.parent.status.slaveDisconnected(self.slavename)

267

268 - def disconnect(self):

269 """Forcibly disconnect the slave. 270 271 This severs the TCP connection and returns a Deferred that will fire 272 (with None) when the connection is probably gone. 273 274 If the slave is still alive, they will probably try to reconnect 275 again in a moment. 276 277 This is called in two circumstances. The first is when a slave is 278 removed from the config file. In this case, when they try to 279 reconnect, they will be rejected as an unknown slave. The second is 280 when we wind up with two connections for the same slave, in which 281 case we disconnect the older connection. 282 """ 283 284 if not self.slave: 285 return defer.succeed(None) 286 log.msg("disconnecting old slave %s now" % self.slavename) 287 # When this Deferred fires, we'll be ready to accept the new slave 288 return self._disconnect(self.slave)

289

290 - def _disconnect(self, slave):

291 # all kinds of teardown will happen as a result of 292 # loseConnection(), but it happens after a reactor iteration or 293 # two. Hook the actual disconnect so we can know when it is safe 294 # to connect the new slave. We have to wait one additional 295 # iteration (with callLater(0)) to make sure the *other* 296 # notifyOnDisconnect handlers have had a chance to run. 297 d = defer.Deferred() 298 299 # notifyOnDisconnect runs the callback with one argument, the 300 # RemoteReference being disconnected. 301 def _disconnected(rref): 302 reactor.callLater(0, d.callback, None)

303 slave.notifyOnDisconnect(_disconnected) 304 tport = slave.broker.transport 305 # this is the polite way to request that a socket be closed 306 tport.loseConnection() 307 try: 308 # but really we don't want to wait for the transmit queue to 309 # drain. The remote end is unlikely to ACK the data, so we'd 310 # probably have to wait for a (20-minute) TCP timeout. 311 #tport._closeSocket() 312 # however, doing _closeSocket (whether before or after 313 # loseConnection) somehow prevents the notifyOnDisconnect 314 # handlers from being run. Bummer. 315 tport.offset = 0 316 tport.dataBuffer = "" 317 except: 318 # however, these hacks are pretty internal, so don't blow up if 319 # they fail or are unavailable 320 log.msg("failed to accelerate the shutdown process") 321 log.msg("waiting for slave to finish disconnecting") 322 323 return d 324

325 - def sendBuilderList(self):

326 our_builders = self.botmaster.getBuildersForSlave(self.slavename) 327 blist = [(b.name, b.slavebuilddir) for b in our_builders] 328 d = self.slave.callRemote("setBuilderList", blist) 329 return d

330

331 - def perspective_keepalive(self):

332 pass

333

334 - def addSlaveBuilder(self, sb):

335 self.slavebuilders[sb.builder_name] = sb

336

337 - def removeSlaveBuilder(self, sb):

338 try: 339 del self.slavebuilders[sb.builder_name] 340 except KeyError: 341 pass

342

343 - def canStartBuild(self):

344 """ 345 I am called when a build is requested to see if this buildslave 346 can start a build. This function can be used to limit overall 347 concurrency on the buildslave. 348 """ 349 # If we're waiting to shutdown gracefully, then we shouldn't 350 # accept any new jobs. 351 if self.slave_status.getGraceful(): 352 return False 353 354 if self.max_builds: 355 active_builders = [sb for sb in self.slavebuilders.values() 356 if sb.isBusy()] 357 if len(active_builders) >= self.max_builds: 358 return False 359 return True

360

361 - def _mail_missing_message(self, subject, text):

362 # first, see if we have a MailNotifier we can use. This gives us a 363 # fromaddr and a relayhost. 364 buildmaster = self.botmaster.parent 365 for st in buildmaster.statusTargets: 366 if isinstance(st, MailNotifier): 367 break 368 else: 369 # if not, they get a default MailNotifier, which always uses SMTP 370 # to localhost and uses a dummy fromaddr of "buildbot". 371 log.msg("buildslave-missing msg using default MailNotifier") 372 st = MailNotifier("buildbot") 373 # now construct the mail 374 375 m = Message() 376 m.set_payload(text) 377 m['Date'] = formatdate(localtime=True) 378 m['Subject'] = subject 379 m['From'] = st.fromaddr 380 recipients = self.notify_on_missing 381 m['To'] = ", ".join(recipients) 382 d = st.sendMessage(m, recipients) 383 # return the Deferred for testing purposes 384 return d

385

386 - def _gracefulChanged(self, graceful):

387 """This is called when our graceful shutdown setting changes""" 388 if graceful: 389 active_builders = [sb for sb in self.slavebuilders.values() 390 if sb.isBusy()] 391 if len(active_builders) == 0: 392 # Shut down! 393 self.shutdown()

394

395 - def shutdown(self):

396 """Shutdown the slave""" 397 # Look for a builder with a remote reference to the client side 398 # slave. If we can find one, then call "shutdown" on the remote 399 # builder, which will cause the slave buildbot process to exit. 400 d = None 401 for b in self.slavebuilders.values(): 402 if b.remote: 403 d = b.remote.callRemote("shutdown") 404 break 405 406 if d: 407 log.msg("Shutting down slave: %s" % self.slavename) 408 # The remote shutdown call will not complete successfully since the 409 # buildbot process exits almost immediately after getting the 410 # shutdown request. 411 # Here we look at the reason why the remote call failed, and if 412 # it's because the connection was lost, that means the slave 413 # shutdown as expected. 414 def _errback(why): 415 if why.check(twisted.spread.pb.PBConnectionLost): 416 log.msg("Lost connection to %s" % self.slavename) 417 else: 418 log.err("Unexpected error when trying to shutdown %s" % self.slavename)

419 d.addErrback(_errback) 420 return d 421 log.err("Couldn't find remote builder to shut down slave") 422 return defer.succeed(None) 423

424 -class BuildSlave(AbstractBuildSlave):

425

426 - def sendBuilderList(self):

427 d = AbstractBuildSlave.sendBuilderList(self) 428 def _sent(slist): 429 dl = [] 430 for name, remote in slist.items(): 431 # use get() since we might have changed our mind since then 432 b = self.botmaster.builders.get(name) 433 if b: 434 d1 = b.attached(self, remote, self.slave_commands) 435 dl.append(d1) 436 return defer.DeferredList(dl)

437 def _set_failed(why): 438 log.msg("BuildSlave.sendBuilderList (%s) failed" % self) 439 log.err(why)

440 # TODO: hang up on them?, without setBuilderList we can't use 441 # them 442 d.addCallbacks(_sent, _set_failed) 443 return d 444

445 - def detached(self, mind):

446 AbstractBuildSlave.detached(self, mind) 447 self.botmaster.slaveLost(self) 448 self.startMissingTimer()

449

450 - def buildFinished(self, sb):

451 """This is called when a build on this slave is finished.""" 452 # If we're gracefully shutting down, and we have no more active 453 # builders, then it's safe to disconnect 454 if self.slave_status.getGraceful(): 455 active_builders = [sb for sb in self.slavebuilders.values() 456 if sb.isBusy()] 457 if len(active_builders) == 0: 458 # Shut down! 459 return self.shutdown() 460 return defer.succeed(None)

461

462 -class AbstractLatentBuildSlave(AbstractBuildSlave):

463 """A build slave that will start up a slave instance when needed. 464 465 To use, subclass and implement start_instance and stop_instance. 466 467 See ec2buildslave.py for a concrete example. Also see the stub example in 468 test/test_slaves.py. 469 """ 470 471 implements(ILatentBuildSlave) 472 473 substantiated = False 474 substantiation_deferred = None 475 build_wait_timer = None 476 _start_result = _shutdown_callback_handle = None 477

478 - def __init__(self, name, password, max_builds=None, 479 notify_on_missing=[], missing_timeout=60*20, 480 build_wait_timeout=60*10, 481 properties={}):

482 AbstractBuildSlave.__init__( 483 self, name, password, max_builds, notify_on_missing, 484 missing_timeout, properties) 485 self.building = set() 486 self.build_wait_timeout = build_wait_timeout

487

488 - def start_instance(self):

489 # responsible for starting instance that will try to connect with 490 # this master. Should return deferred. Problems should use an 491 # errback. 492 raise NotImplementedError

493

494 - def stop_instance(self, fast=False):

495 # responsible for shutting down instance. 496 raise NotImplementedError

497

498 - def substantiate(self, sb):

499 if self.substantiated: 500 self._clearBuildWaitTimer() 501 self._setBuildWaitTimer() 502 return defer.succeed(self) 503 if self.substantiation_deferred is None: 504 if self.parent and not self.missing_timer: 505 # start timer. if timer times out, fail deferred 506 self.missing_timer = reactor.callLater( 507 self.missing_timeout, 508 self._substantiation_failed, defer.TimeoutError()) 509 self.substantiation_deferred = defer.Deferred() 510 if self.slave is None: 511 self._substantiate() # start up instance 512 # else: we're waiting for an old one to detach. the _substantiate 513 # will be done in ``detached`` below. 514 return self.substantiation_deferred

515

516 - def _substantiate(self):

517 # register event trigger 518 d = self.start_instance() 519 self._shutdown_callback_handle = reactor.addSystemEventTrigger( 520 'before', 'shutdown', self._soft_disconnect, fast=True) 521 def stash_reply(result): 522 self._start_result = result

523 def clean_up(failure): 524 if self.missing_timer is not None: 525 self.missing_timer.cancel() 526 self._substantiation_failed(failure) 527 if self._shutdown_callback_handle is not None: 528 handle = self._shutdown_callback_handle 529 del self._shutdown_callback_handle 530 reactor.removeSystemEventTrigger(handle) 531 return failure

532 d.addCallbacks(stash_reply, clean_up) 533 return d 534

535 - def attached(self, bot):

536 if self.substantiation_deferred is None: 537 msg = 'Slave %s received connection while not trying to ' \ 538 'substantiate. Disconnecting.' % (self.slavename,) 539 log.msg(msg) 540 self._disconnect(bot) 541 return defer.fail(RuntimeError(msg)) 542 return AbstractBuildSlave.attached(self, bot)

543

544 - def detached(self, mind):

545 AbstractBuildSlave.detached(self, mind) 546 if self.substantiation_deferred is not None: 547 self._substantiate()

548

549 - def _substantiation_failed(self, failure):

550 d = self.substantiation_deferred 551 self.substantiation_deferred = None 552 self.missing_timer = None 553 d.errback(failure) 554 self.insubstantiate() 555 # notify people, but only if we're still in the config 556 if not self.parent or not self.notify_on_missing: 557 return 558 559 buildmaster = self.botmaster.parent 560 status = buildmaster.getStatus() 561 text = "The Buildbot working for '%s'\n" % status.getProjectName() 562 text += ("has noticed that the latent buildslave named %s \n" % 563 self.slavename) 564 text += "never substantiated after a request\n" 565 text += "\n" 566 text += ("The request was made at %s (buildmaster-local time)\n" % 567 time.ctime(time.time() - self.missing_timeout)) # approx 568 text += "\n" 569 text += "Sincerely,\n" 570 text += " The Buildbot\n" 571 text += " %s\n" % status.getProjectURL() 572 subject = "Buildbot: buildslave %s never substantiated" % self.slavename 573 return self._mail_missing_message(subject, text)

574

575 - def buildStarted(self, sb):

576 assert self.substantiated 577 self._clearBuildWaitTimer() 578 self.building.add(sb.builder_name)

579

580 - def buildFinished(self, sb):

581 self.building.remove(sb.builder_name) 582 if not self.building: 583 self._setBuildWaitTimer()

584

585 - def _clearBuildWaitTimer(self):

586 if self.build_wait_timer is not None: 587 if self.build_wait_timer.active(): 588 self.build_wait_timer.cancel() 589 self.build_wait_timer = None

590

591 - def _setBuildWaitTimer(self):

592 self._clearBuildWaitTimer() 593 self.build_wait_timer = reactor.callLater( 594 self.build_wait_timeout, self._soft_disconnect)

595

596 - def insubstantiate(self, fast=False):

597 self._clearBuildWaitTimer() 598 d = self.stop_instance(fast) 599 if self._shutdown_callback_handle is not None: 600 handle = self._shutdown_callback_handle 601 del self._shutdown_callback_handle 602 reactor.removeSystemEventTrigger(handle) 603 self.substantiated = False 604 self.building.clear() # just to be sure 605 return d

606

607 - def _soft_disconnect(self, fast=False):

608 d = AbstractBuildSlave.disconnect(self) 609 if self.slave is not None: 610 # this could be called when the slave needs to shut down, such as 611 # in BotMaster.removeSlave, *or* when a new slave requests a 612 # connection when we already have a slave. It's not clear what to 613 # do in the second case: this shouldn't happen, and if it 614 # does...if it's a latent slave, shutting down will probably kill 615 # something we want...but we can't know what the status is. So, 616 # here, we just do what should be appropriate for the first case, 617 # and put our heads in the sand for the second, at least for now. 618 # The best solution to the odd situation is removing it as a 619 # possibilty: make the master in charge of connecting to the 620 # slave, rather than vice versa. TODO. 621 d = defer.DeferredList([d, self.insubstantiate(fast)]) 622 else: 623 if self.substantiation_deferred is not None: 624 # unlike the previous block, we don't expect this situation when 625 # ``attached`` calls ``disconnect``, only when we get a simple 626 # request to "go away". 627 self.substantiation_deferred.errback() 628 self.substantiation_deferred = None 629 if self.missing_timer: 630 self.missing_timer.cancel() 631 self.missing_timer = None 632 self.stop_instance() 633 return d

634

635 - def disconnect(self):

636 d = self._soft_disconnect() 637 # this removes the slave from all builders. It won't come back 638 # without a restart (or maybe a sighup) 639 self.botmaster.slaveLost(self)

640

641 - def stopService(self):

642 res = defer.maybeDeferred(AbstractBuildSlave.stopService, self) 643 if self.slave is not None: 644 d = self._soft_disconnect() 645 res = defer.DeferredList([res, d]) 646 return res

647

648 - def updateSlave(self):

649 """Called to add or remove builders after the slave has connected. 650 651 Also called after botmaster's builders are initially set. 652 653 @return: a Deferred that indicates when an attached slave has 654 accepted the new builders and/or released the old ones.""" 655 for b in self.botmaster.getBuildersForSlave(self.slavename): 656 if b.name not in self.slavebuilders: 657 b.addLatentSlave(self) 658 return AbstractBuildSlave.updateSlave(self)

659

660 - def sendBuilderList(self):

661 d = AbstractBuildSlave.sendBuilderList(self) 662 def _sent(slist): 663 dl = [] 664 for name, remote in slist.items(): 665 # use get() since we might have changed our mind since then. 666 # we're checking on the builder in addition to the 667 # slavebuilders out of a bit of paranoia. 668 b = self.botmaster.builders.get(name) 669 sb = self.slavebuilders.get(name) 670 if b and sb: 671 d1 = sb.attached(self, remote, self.slave_commands) 672 dl.append(d1) 673 return defer.DeferredList(dl)

674 def _set_failed(why): 675 log.msg("BuildSlave.sendBuilderList (%s) failed" % self) 676 log.err(why) 677 # TODO: hang up on them?, without setBuilderList we can't use 678 # them 679 if self.substantiation_deferred: 680 self.substantiation_deferred.errback() 681 self.substantiation_deferred = None 682 if self.missing_timer: 683 self.missing_timer.cancel() 684 self.missing_timer = None 685 # TODO: maybe log? send an email? 686 return why 687 d.addCallbacks(_sent, _set_failed) 688 def _substantiated(res): 689 self.substantiated = True 690 if self.substantiation_deferred: 691 d = self.substantiation_deferred 692 del self.substantiation_deferred 693 res = self._start_result 694 del self._start_result 695 d.callback(res) 696 # note that the missing_timer is already handled within 697 # ``attached`` 698 if not self.building: 699 self._setBuildWaitTimer() 700 d.addCallback(_substantiated) 701 return d 702

Source Code for Module buildbot.buildslave