buildbot.buildslave

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Portions Copyright Buildbot Team Members 15 # Portions Copyright Canonical Ltd. 2009 16 17 import time 18 from email.Message import Message 19 from email.Utils import formatdate 20 from zope.interface import implements 21 from twisted.python import log, failure 22 from twisted.internet import defer, reactor 23 from twisted.application import service 24 from twisted.spread import pb 25 from twisted.python.reflect import namedModule 26 27 from buildbot.status.slave import SlaveStatus 28 from buildbot.status.mail import MailNotifier 29 from buildbot.process import metrics 30 from buildbot.interfaces import IBuildSlave, ILatentBuildSlave 31 from buildbot.process.properties import Properties 32 from buildbot.locks import LockAccess

33 34 -class AbstractBuildSlave(pb.Avatar, service.MultiService):

35 """This is the master-side representative for a remote buildbot slave. 36 There is exactly one for each slave described in the config file (the 37 c['slaves'] list). When buildbots connect in (.attach), they get a 38 reference to this instance. The BotMaster object is stashed as the 39 .botmaster attribute. The BotMaster is also our '.parent' Service. 40 41 I represent a build slave -- a remote machine capable of 42 running builds. I am instantiated by the configuration file, and can be 43 subclassed to add extra functionality.""" 44 45 implements(IBuildSlave) 46 keepalive_timer = None 47 keepalive_interval = None 48

49 - def __init__(self, name, password, max_builds=None, 50 notify_on_missing=[], missing_timeout=3600, 51 properties={}, locks=None, keepalive_interval=3600):

52 """ 53 @param name: botname this machine will supply when it connects 54 @param password: password this machine will supply when 55 it connects 56 @param max_builds: maximum number of simultaneous builds that will 57 be run concurrently on this buildslave (the 58 default is None for no limit) 59 @param properties: properties that will be applied to builds run on 60 this slave 61 @type properties: dictionary 62 @param locks: A list of locks that must be acquired before this slave 63 can be used 64 @type locks: dictionary 65 """ 66 service.MultiService.__init__(self) 67 self.slavename = name 68 self.password = password 69 self.botmaster = None # no buildmaster yet 70 self.slave_status = SlaveStatus(name) 71 self.slave = None # a RemoteReference to the Bot, when connected 72 self.slave_commands = None 73 self.slavebuilders = {} 74 self.max_builds = max_builds 75 self.access = [] 76 if locks: 77 self.access = locks 78 79 self.properties = Properties() 80 self.properties.update(properties, "BuildSlave") 81 self.properties.setProperty("slavename", name, "BuildSlave") 82 83 self.lastMessageReceived = 0 84 if isinstance(notify_on_missing, str): 85 notify_on_missing = [notify_on_missing] 86 self.notify_on_missing = notify_on_missing 87 for i in notify_on_missing: 88 assert isinstance(i, str) 89 self.missing_timeout = missing_timeout 90 self.missing_timer = None 91 self.keepalive_interval = keepalive_interval 92 93 self._old_builder_list = None

94

95 - def identity(self):

96 """ 97 Return a tuple describing this slave. After reconfiguration a 98 new slave with the same identity will update this one, rather 99 than replacing it, thereby avoiding an interruption of current 100 activity. 101 """ 102 return (self.slavename, self.password, 103 '%s.%s' % (self.__class__.__module__, 104 self.__class__.__name__))

105

106 - def update(self, new):

107 """ 108 Given a new BuildSlave, configure this one identically. Because 109 BuildSlave objects are remotely referenced, we can't replace them 110 without disconnecting the slave, yet there's no reason to do that. 111 """ 112 # the reconfiguration logic should guarantee this: 113 assert self.slavename == new.slavename 114 assert self.password == new.password 115 assert self.identity() == new.identity() 116 self.max_builds = new.max_builds 117 self.access = new.access 118 self.notify_on_missing = new.notify_on_missing 119 self.missing_timeout = new.missing_timeout 120 self.keepalive_interval = new.keepalive_interval 121 122 self.properties = Properties() 123 self.properties.updateFromProperties(new.properties) 124 125 if self.botmaster: 126 self.updateLocks()

127

128 - def __repr__(self):

129 if self.botmaster: 130 builders = self.botmaster.getBuildersForSlave(self.slavename) 131 return "<%s '%s', current builders: %s>" % \ 132 (self.__class__.__name__, self.slavename, 133 ','.join(map(lambda b: b.name, builders))) 134 else: 135 return "<%s '%s', (no builders yet)>" % \ 136 (self.__class__.__name__, self.slavename)

137

138 - def updateLocks(self):

139 # convert locks into their real form 140 locks = [] 141 for access in self.access: 142 if not isinstance(access, LockAccess): 143 access = access.defaultAccess() 144 lock = self.botmaster.getLockByID(access.lockid) 145 locks.append((lock, access)) 146 self.locks = [(l.getLock(self), la) for l, la in locks]

147

148 - def locksAvailable(self):

149 """ 150 I am called to see if all the locks I depend on are available, 151 in which I return True, otherwise I return False 152 """ 153 if not self.locks: 154 return True 155 for lock, access in self.locks: 156 if not lock.isAvailable(access): 157 return False 158 return True

159

160 - def acquireLocks(self):

161 """ 162 I am called when a build is preparing to run. I try to claim all 163 the locks that are needed for a build to happen. If I can't, then 164 my caller should give up the build and try to get another slave 165 to look at it. 166 """ 167 log.msg("acquireLocks(slave %s, locks %s)" % (self, self.locks)) 168 if not self.locksAvailable(): 169 log.msg("slave %s can't lock, giving up" % (self, )) 170 return False 171 # all locks are available, claim them all 172 for lock, access in self.locks: 173 lock.claim(self, access) 174 return True

175

176 - def releaseLocks(self):

177 """ 178 I am called to release any locks after a build has finished 179 """ 180 log.msg("releaseLocks(%s): %s" % (self, self.locks)) 181 for lock, access in self.locks: 182 lock.release(self, access)

183

184 - def setBotmaster(self, botmaster):

185 assert not self.botmaster, "BuildSlave already has a botmaster" 186 self.botmaster = botmaster 187 self.updateLocks() 188 self.startMissingTimer()

189

190 - def stopMissingTimer(self):

191 if self.missing_timer: 192 self.missing_timer.cancel() 193 self.missing_timer = None

194

195 - def startMissingTimer(self):

196 if self.notify_on_missing and self.missing_timeout and self.parent: 197 self.stopMissingTimer() # in case it's already running 198 self.missing_timer = reactor.callLater(self.missing_timeout, 199 self._missing_timer_fired)

200

201 - def doKeepalive(self):

202 self.keepalive_timer = reactor.callLater(self.keepalive_interval, 203 self.doKeepalive) 204 if not self.slave: 205 return 206 d = self.slave.callRemote("print", "Received keepalive from master") 207 d.addErrback(log.msg, "Keepalive failed for '%s'" % (self.slavename, ))

208

209 - def stopKeepaliveTimer(self):

210 if self.keepalive_timer: 211 self.keepalive_timer.cancel()

212

213 - def startKeepaliveTimer(self):

214 assert self.keepalive_interval 215 log.msg("Starting buildslave keepalive timer for '%s'" % \ 216 (self.slavename, )) 217 self.doKeepalive()

218

219 - def recordConnectTime(self):

220 if self.slave_status: 221 self.slave_status.recordConnectTime()

222

223 - def isConnected(self):

224 return self.slave

225

226 - def _missing_timer_fired(self):

227 self.missing_timer = None 228 # notify people, but only if we're still in the config 229 if not self.parent: 230 return 231 232 buildmaster = self.botmaster.parent 233 status = buildmaster.getStatus() 234 text = "The Buildbot working for '%s'\n" % status.getTitle() 235 text += ("has noticed that the buildslave named %s went away\n" % 236 self.slavename) 237 text += "\n" 238 text += ("It last disconnected at %s (buildmaster-local time)\n" % 239 time.ctime(time.time() - self.missing_timeout)) # approx 240 text += "\n" 241 text += "The admin on record (as reported by BUILDSLAVE:info/admin)\n" 242 text += "was '%s'.\n" % self.slave_status.getAdmin() 243 text += "\n" 244 text += "Sincerely,\n" 245 text += " The Buildbot\n" 246 text += " %s\n" % status.getTitleURL() 247 subject = "Buildbot: buildslave %s was lost" % self.slavename 248 return self._mail_missing_message(subject, text)

249 250

251 - def updateSlave(self):

252 """Called to add or remove builders after the slave has connected. 253 254 @return: a Deferred that indicates when an attached slave has 255 accepted the new builders and/or released the old ones.""" 256 if self.slave: 257 return self.sendBuilderList() 258 else: 259 return defer.succeed(None)

260

261 - def updateSlaveStatus(self, buildStarted=None, buildFinished=None):

262 if buildStarted: 263 self.slave_status.buildStarted(buildStarted) 264 if buildFinished: 265 self.slave_status.buildFinished(buildFinished)

266 267 @metrics.countMethod('AbstractBuildSlave.attached()')

268 - def attached(self, bot):

269 """This is called when the slave connects. 270 271 @return: a Deferred that fires when the attachment is complete 272 """ 273 274 # the botmaster should ensure this. 275 assert not self.isConnected() 276 277 metrics.MetricCountEvent.log("AbstractBuildSlave.attached_slaves", 1) 278 279 # now we go through a sequence of calls, gathering information, then 280 # tell the Botmaster that it can finally give this slave to all the 281 # Builders that care about it. 282 283 # we accumulate slave information in this 'state' dictionary, then 284 # set it atomically if we make it far enough through the process 285 state = {} 286 287 # Reset graceful shutdown status 288 self.slave_status.setGraceful(False) 289 # We want to know when the graceful shutdown flag changes 290 self.slave_status.addGracefulWatcher(self._gracefulChanged) 291 292 d = defer.succeed(None) 293 def _log_attachment_on_slave(res): 294 d1 = bot.callRemote("print", "attached") 295 d1.addErrback(lambda why: None) 296 return d1

297 d.addCallback(_log_attachment_on_slave) 298 299 def _get_info(res): 300 d1 = bot.callRemote("getSlaveInfo") 301 def _got_info(info): 302 log.msg("Got slaveinfo from '%s'" % self.slavename) 303 # TODO: info{} might have other keys 304 state["admin"] = info.get("admin") 305 state["host"] = info.get("host") 306 state["access_uri"] = info.get("access_uri", None) 307 state["slave_environ"] = info.get("environ", {}) 308 state["slave_basedir"] = info.get("basedir", None) 309 state["slave_system"] = info.get("system", None)

310 def _info_unavailable(why): 311 why.trap(pb.NoSuchMethod) 312 # maybe an old slave, doesn't implement remote_getSlaveInfo 313 log.msg("BuildSlave.info_unavailable") 314 log.err(why) 315 d1.addCallbacks(_got_info, _info_unavailable) 316 return d1 317 d.addCallback(_get_info) 318 self.startKeepaliveTimer() 319 320 def _get_version(res): 321 d = bot.callRemote("getVersion") 322 def _got_version(version): 323 state["version"] = version 324 def _version_unavailable(why): 325 why.trap(pb.NoSuchMethod) 326 # probably an old slave 327 state["version"] = '(unknown)' 328 d.addCallbacks(_got_version, _version_unavailable) 329 return d 330 d.addCallback(_get_version) 331 332 def _get_commands(res): 333 d1 = bot.callRemote("getCommands") 334 def _got_commands(commands): 335 state["slave_commands"] = commands 336 def _commands_unavailable(why): 337 # probably an old slave 338 log.msg("BuildSlave._commands_unavailable") 339 if why.check(AttributeError): 340 return 341 log.err(why) 342 d1.addCallbacks(_got_commands, _commands_unavailable) 343 return d1 344 d.addCallback(_get_commands) 345 346 def _accept_slave(res): 347 self.slave_status.setAdmin(state.get("admin")) 348 self.slave_status.setHost(state.get("host")) 349 self.slave_status.setAccessURI(state.get("access_uri")) 350 self.slave_status.setVersion(state.get("version")) 351 self.slave_status.setConnected(True) 352 self.slave_commands = state.get("slave_commands") 353 self.slave_environ = state.get("slave_environ") 354 self.slave_basedir = state.get("slave_basedir") 355 self.slave_system = state.get("slave_system") 356 self.slave = bot 357 if self.slave_system == "win32": 358 self.path_module = namedModule("win32path") 359 else: 360 # most eveything accepts / as separator, so posix should be a 361 # reasonable fallback 362 self.path_module = namedModule("posixpath") 363 log.msg("bot attached") 364 self.messageReceivedFromSlave() 365 self.stopMissingTimer() 366 self.botmaster.parent.status.slaveConnected(self.slavename) 367 368 return self.updateSlave() 369 d.addCallback(_accept_slave) 370 d.addCallback(lambda _: 371 self.botmaster.maybeStartBuildsForSlave(self.slavename)) 372 373 # Finally, the slave gets a reference to this BuildSlave. They 374 # receive this later, after we've started using them. 375 d.addCallback(lambda _: self) 376 return d 377

378 - def messageReceivedFromSlave(self):

379 now = time.time() 380 self.lastMessageReceived = now 381 self.slave_status.setLastMessageReceived(now)

382

383 - def detached(self, mind):

384 metrics.MetricCountEvent.log("AbstractBuildSlave.attached_slaves", -1) 385 self.slave = None 386 self._old_builder_list = [] 387 self.slave_status.removeGracefulWatcher(self._gracefulChanged) 388 self.slave_status.setConnected(False) 389 log.msg("BuildSlave.detached(%s)" % self.slavename) 390 self.botmaster.parent.status.slaveDisconnected(self.slavename) 391 self.stopKeepaliveTimer()

392

393 - def disconnect(self):

394 """Forcibly disconnect the slave. 395 396 This severs the TCP connection and returns a Deferred that will fire 397 (with None) when the connection is probably gone. 398 399 If the slave is still alive, they will probably try to reconnect 400 again in a moment. 401 402 This is called in two circumstances. The first is when a slave is 403 removed from the config file. In this case, when they try to 404 reconnect, they will be rejected as an unknown slave. The second is 405 when we wind up with two connections for the same slave, in which 406 case we disconnect the older connection. 407 """ 408 409 if not self.slave: 410 return defer.succeed(None) 411 log.msg("disconnecting old slave %s now" % self.slavename) 412 # When this Deferred fires, we'll be ready to accept the new slave 413 return self._disconnect(self.slave)

414

415 - def _disconnect(self, slave):

416 # all kinds of teardown will happen as a result of 417 # loseConnection(), but it happens after a reactor iteration or 418 # two. Hook the actual disconnect so we can know when it is safe 419 # to connect the new slave. We have to wait one additional 420 # iteration (with callLater(0)) to make sure the *other* 421 # notifyOnDisconnect handlers have had a chance to run. 422 d = defer.Deferred() 423 424 # notifyOnDisconnect runs the callback with one argument, the 425 # RemoteReference being disconnected. 426 def _disconnected(rref): 427 reactor.callLater(0, d.callback, None)

428 slave.notifyOnDisconnect(_disconnected) 429 tport = slave.broker.transport 430 # this is the polite way to request that a socket be closed 431 tport.loseConnection() 432 try: 433 # but really we don't want to wait for the transmit queue to 434 # drain. The remote end is unlikely to ACK the data, so we'd 435 # probably have to wait for a (20-minute) TCP timeout. 436 #tport._closeSocket() 437 # however, doing _closeSocket (whether before or after 438 # loseConnection) somehow prevents the notifyOnDisconnect 439 # handlers from being run. Bummer. 440 tport.offset = 0 441 tport.dataBuffer = "" 442 except: 443 # however, these hacks are pretty internal, so don't blow up if 444 # they fail or are unavailable 445 log.msg("failed to accelerate the shutdown process") 446 log.msg("waiting for slave to finish disconnecting") 447 448 return d 449

450 - def sendBuilderList(self):

451 our_builders = self.botmaster.getBuildersForSlave(self.slavename) 452 blist = [(b.name, b.slavebuilddir) for b in our_builders] 453 if blist == self._old_builder_list: 454 log.msg("Builder list is unchanged; not calling setBuilderList") 455 return defer.succeed(None) 456 457 d = self.slave.callRemote("setBuilderList", blist) 458 def sentBuilderList(ign): 459 self._old_builder_list = blist 460 return ign

461 d.addCallback(sentBuilderList) 462 return d 463

464 - def perspective_keepalive(self):

465 self.messageReceivedFromSlave()

466

467 - def perspective_shutdown(self):

468 log.msg("slave %s wants to shut down" % self.slavename) 469 self.slave_status.setGraceful(True)

470

471 - def addSlaveBuilder(self, sb):

472 self.slavebuilders[sb.builder_name] = sb

473

474 - def removeSlaveBuilder(self, sb):

475 try: 476 del self.slavebuilders[sb.builder_name] 477 except KeyError: 478 pass

479

480 - def buildFinished(self, sb):

481 """This is called when a build on this slave is finished.""" 482 self.botmaster.maybeStartBuildsForSlave(self.slavename)

483

484 - def canStartBuild(self):

485 """ 486 I am called when a build is requested to see if this buildslave 487 can start a build. This function can be used to limit overall 488 concurrency on the buildslave. 489 """ 490 # If we're waiting to shutdown gracefully, then we shouldn't 491 # accept any new jobs. 492 if self.slave_status.getGraceful(): 493 return False 494 495 if self.max_builds: 496 active_builders = [sb for sb in self.slavebuilders.values() 497 if sb.isBusy()] 498 if len(active_builders) >= self.max_builds: 499 return False 500 501 if not self.locksAvailable(): 502 return False 503 504 return True

505

506 - def _mail_missing_message(self, subject, text):

507 # first, see if we have a MailNotifier we can use. This gives us a 508 # fromaddr and a relayhost. 509 buildmaster = self.botmaster.parent 510 for st in buildmaster.statusTargets: 511 if isinstance(st, MailNotifier): 512 break 513 else: 514 # if not, they get a default MailNotifier, which always uses SMTP 515 # to localhost and uses a dummy fromaddr of "buildbot". 516 log.msg("buildslave-missing msg using default MailNotifier") 517 st = MailNotifier("buildbot") 518 # now construct the mail 519 520 m = Message() 521 m.set_payload(text) 522 m['Date'] = formatdate(localtime=True) 523 m['Subject'] = subject 524 m['From'] = st.fromaddr 525 recipients = self.notify_on_missing 526 m['To'] = ", ".join(recipients) 527 d = st.sendMessage(m, recipients) 528 # return the Deferred for testing purposes 529 return d

530

531 - def _gracefulChanged(self, graceful):

532 """This is called when our graceful shutdown setting changes""" 533 self.maybeShutdown()

534 535 @defer.deferredGenerator

536 - def shutdown(self):

537 """Shutdown the slave""" 538 if not self.slave: 539 log.msg("no remote; slave is already shut down") 540 return 541 542 # First, try the "new" way - calling our own remote's shutdown 543 # method. The method was only added in 0.8.3, so ignore NoSuchMethod 544 # failures. 545 def new_way(): 546 d = self.slave.callRemote('shutdown') 547 d.addCallback(lambda _ : True) # successful shutdown request 548 def check_nsm(f): 549 f.trap(pb.NoSuchMethod) 550 return False # fall through to the old way

551 d.addErrback(check_nsm) 552 def check_connlost(f): 553 f.trap(pb.PBConnectionLost) 554 return True # the slave is gone, so call it finished 555 d.addErrback(check_connlost) 556 return d 557 558 wfd = defer.waitForDeferred(new_way()) 559 yield wfd 560 if wfd.getResult(): 561 return # done! 562 563 # Now, the old way. Look for a builder with a remote reference to the 564 # client side slave. If we can find one, then call "shutdown" on the 565 # remote builder, which will cause the slave buildbot process to exit. 566 def old_way(): 567 d = None 568 for b in self.slavebuilders.values(): 569 if b.remote: 570 d = b.remote.callRemote("shutdown") 571 break 572 573 if d: 574 log.msg("Shutting down (old) slave: %s" % self.slavename) 575 # The remote shutdown call will not complete successfully since the 576 # buildbot process exits almost immediately after getting the 577 # shutdown request. 578 # Here we look at the reason why the remote call failed, and if 579 # it's because the connection was lost, that means the slave 580 # shutdown as expected. 581 def _errback(why): 582 if why.check(pb.PBConnectionLost): 583 log.msg("Lost connection to %s" % self.slavename) 584 else: 585 log.err("Unexpected error when trying to shutdown %s" % self.slavename) 586 d.addErrback(_errback) 587 return d 588 log.err("Couldn't find remote builder to shut down slave") 589 return defer.succeed(None) 590 wfd = defer.waitForDeferred(old_way()) 591 yield wfd 592 wfd.getResult() 593

594 - def maybeShutdown(self):

595 """Shut down this slave if it has been asked to shut down gracefully, 596 and has no active builders.""" 597 if not self.slave_status.getGraceful(): 598 return 599 active_builders = [sb for sb in self.slavebuilders.values() 600 if sb.isBusy()] 601 if active_builders: 602 return 603 d = self.shutdown() 604 d.addErrback(log.err, 'error while shutting down slave')

605

606 -class BuildSlave(AbstractBuildSlave):

607

608 - def sendBuilderList(self):

609 d = AbstractBuildSlave.sendBuilderList(self) 610 def _sent(slist): 611 # Nothing has changed, so don't need to re-attach to everything 612 if not slist: 613 return 614 dl = [] 615 for name, remote in slist.items(): 616 # use get() since we might have changed our mind since then 617 b = self.botmaster.builders.get(name) 618 if b: 619 d1 = b.attached(self, remote, self.slave_commands) 620 dl.append(d1) 621 return defer.DeferredList(dl)

622 def _set_failed(why): 623 log.msg("BuildSlave.sendBuilderList (%s) failed" % self) 624 log.err(why)

625 # TODO: hang up on them?, without setBuilderList we can't use 626 # them 627 d.addCallbacks(_sent, _set_failed) 628 return d 629

630 - def detached(self, mind):

631 AbstractBuildSlave.detached(self, mind) 632 self.botmaster.slaveLost(self) 633 self.startMissingTimer()

634

635 - def buildFinished(self, sb):

636 """This is called when a build on this slave is finished.""" 637 AbstractBuildSlave.buildFinished(self, sb) 638 639 # If we're gracefully shutting down, and we have no more active 640 # builders, then it's safe to disconnect 641 self.maybeShutdown()

642

643 -class AbstractLatentBuildSlave(AbstractBuildSlave):

644 """A build slave that will start up a slave instance when needed. 645 646 To use, subclass and implement start_instance and stop_instance. 647 648 See ec2buildslave.py for a concrete example. Also see the stub example in 649 test/test_slaves.py. 650 """ 651 652 implements(ILatentBuildSlave) 653 654 substantiated = False 655 substantiation_deferred = None 656 substantiation_build = None 657 build_wait_timer = None 658 _shutdown_callback_handle = None 659

660 - def __init__(self, name, password, max_builds=None, 661 notify_on_missing=[], missing_timeout=60*20, 662 build_wait_timeout=60*10, 663 properties={}, locks=None):

664 AbstractBuildSlave.__init__( 665 self, name, password, max_builds, notify_on_missing, 666 missing_timeout, properties, locks) 667 self.building = set() 668 self.build_wait_timeout = build_wait_timeout

669

670 - def start_instance(self, build):

671 # responsible for starting instance that will try to connect with this 672 # master. Should return deferred with either True (instance started) 673 # or False (instance not started, so don't run a build here). Problems 674 # should use an errback. 675 raise NotImplementedError

676

677 - def stop_instance(self, fast=False):

678 # responsible for shutting down instance. 679 raise NotImplementedError

680

681 - def substantiate(self, sb, build):

682 if self.substantiated: 683 self._clearBuildWaitTimer() 684 self._setBuildWaitTimer() 685 return defer.succeed(True) 686 if self.substantiation_deferred is None: 687 if self.parent and not self.missing_timer: 688 # start timer. if timer times out, fail deferred 689 self.missing_timer = reactor.callLater( 690 self.missing_timeout, 691 self._substantiation_failed, defer.TimeoutError()) 692 self.substantiation_deferred = defer.Deferred() 693 self.substantiation_build = build 694 if self.slave is None: 695 d = self._substantiate(build) # start up instance 696 d.addErrback(log.err, "while substantiating") 697 # else: we're waiting for an old one to detach. the _substantiate 698 # will be done in ``detached`` below. 699 return self.substantiation_deferred

700

701 - def _substantiate(self, build):

702 # register event trigger 703 d = self.start_instance(build) 704 self._shutdown_callback_handle = reactor.addSystemEventTrigger( 705 'before', 'shutdown', self._soft_disconnect, fast=True) 706 def start_instance_result(result): 707 # If we don't report success, then preparation failed. 708 if not result: 709 log.msg("Slave '%s' doesn not want to substantiate at this time" % (self.slavename,)) 710 d = self.substantiation_deferred 711 self.substantiation_deferred = None 712 d.callback(False) 713 return result

714 def clean_up(failure): 715 if self.missing_timer is not None: 716 self.missing_timer.cancel() 717 self._substantiation_failed(failure) 718 if self._shutdown_callback_handle is not None: 719 handle = self._shutdown_callback_handle 720 del self._shutdown_callback_handle 721 reactor.removeSystemEventTrigger(handle) 722 return failure

723 d.addCallbacks(start_instance_result, clean_up) 724 return d 725

726 - def attached(self, bot):

727 if self.substantiation_deferred is None: 728 msg = 'Slave %s received connection while not trying to ' \ 729 'substantiate. Disconnecting.' % (self.slavename,) 730 log.msg(msg) 731 self._disconnect(bot) 732 return defer.fail(RuntimeError(msg)) 733 return AbstractBuildSlave.attached(self, bot)

734

735 - def detached(self, mind):

736 AbstractBuildSlave.detached(self, mind) 737 if self.substantiation_deferred is not None: 738 d = self._substantiate(self.substantiation_build) 739 d.addErrback(log.err, 'while re-substantiating')

740

741 - def _substantiation_failed(self, failure):

742 self.missing_timer = None 743 if self.substantiation_deferred: 744 d = self.substantiation_deferred 745 self.substantiation_deferred = None 746 self.substantiation_build = None 747 d.errback(failure) 748 self.insubstantiate() 749 # notify people, but only if we're still in the config 750 if not self.parent or not self.notify_on_missing: 751 return 752 753 buildmaster = self.botmaster.parent 754 status = buildmaster.getStatus() 755 text = "The Buildbot working for '%s'\n" % status.getTitle() 756 text += ("has noticed that the latent buildslave named %s \n" % 757 self.slavename) 758 text += "never substantiated after a request\n" 759 text += "\n" 760 text += ("The request was made at %s (buildmaster-local time)\n" % 761 time.ctime(time.time() - self.missing_timeout)) # approx 762 text += "\n" 763 text += "Sincerely,\n" 764 text += " The Buildbot\n" 765 text += " %s\n" % status.getTitleURL() 766 subject = "Buildbot: buildslave %s never substantiated" % self.slavename 767 return self._mail_missing_message(subject, text)

768

769 - def buildStarted(self, sb):

770 assert self.substantiated 771 self._clearBuildWaitTimer() 772 self.building.add(sb.builder_name)

773

774 - def buildFinished(self, sb):

775 AbstractBuildSlave.buildFinished(self, sb) 776 777 self.building.remove(sb.builder_name) 778 if not self.building: 779 self._setBuildWaitTimer()

780

781 - def _clearBuildWaitTimer(self):

782 if self.build_wait_timer is not None: 783 if self.build_wait_timer.active(): 784 self.build_wait_timer.cancel() 785 self.build_wait_timer = None

786

787 - def _setBuildWaitTimer(self):

788 self._clearBuildWaitTimer() 789 self.build_wait_timer = reactor.callLater( 790 self.build_wait_timeout, self._soft_disconnect)

791

792 - def insubstantiate(self, fast=False):

793 self._clearBuildWaitTimer() 794 d = self.stop_instance(fast) 795 if self._shutdown_callback_handle is not None: 796 handle = self._shutdown_callback_handle 797 del self._shutdown_callback_handle 798 reactor.removeSystemEventTrigger(handle) 799 self.substantiated = False 800 self.building.clear() # just to be sure 801 return d

802

803 - def _soft_disconnect(self, fast=False):

804 d = AbstractBuildSlave.disconnect(self) 805 if self.slave is not None: 806 # this could be called when the slave needs to shut down, such as 807 # in BotMaster.removeSlave, *or* when a new slave requests a 808 # connection when we already have a slave. It's not clear what to 809 # do in the second case: this shouldn't happen, and if it 810 # does...if it's a latent slave, shutting down will probably kill 811 # something we want...but we can't know what the status is. So, 812 # here, we just do what should be appropriate for the first case, 813 # and put our heads in the sand for the second, at least for now. 814 # The best solution to the odd situation is removing it as a 815 # possibilty: make the master in charge of connecting to the 816 # slave, rather than vice versa. TODO. 817 d = defer.DeferredList([d, self.insubstantiate(fast)]) 818 else: 819 if self.substantiation_deferred is not None: 820 # unlike the previous block, we don't expect this situation when 821 # ``attached`` calls ``disconnect``, only when we get a simple 822 # request to "go away". 823 d = self.substantiation_deferred 824 self.substantiation_deferred = None 825 self.substantiation_build = None 826 d.errback(failure.Failure( 827 RuntimeError("soft disconnect aborted substantiation"))) 828 if self.missing_timer: 829 self.missing_timer.cancel() 830 self.missing_timer = None 831 self.stop_instance() 832 return d

833

834 - def disconnect(self):

835 # This returns a Deferred but we don't use it 836 self._soft_disconnect() 837 # this removes the slave from all builders. It won't come back 838 # without a restart (or maybe a sighup) 839 self.botmaster.slaveLost(self)

840

841 - def stopService(self):

842 res = defer.maybeDeferred(AbstractBuildSlave.stopService, self) 843 if self.slave is not None: 844 d = self._soft_disconnect() 845 res = defer.DeferredList([res, d]) 846 return res

847

848 - def updateSlave(self):

849 """Called to add or remove builders after the slave has connected. 850 851 Also called after botmaster's builders are initially set. 852 853 @return: a Deferred that indicates when an attached slave has 854 accepted the new builders and/or released the old ones.""" 855 for b in self.botmaster.getBuildersForSlave(self.slavename): 856 if b.name not in self.slavebuilders: 857 b.addLatentSlave(self) 858 return AbstractBuildSlave.updateSlave(self)

859

860 - def sendBuilderList(self):

861 d = AbstractBuildSlave.sendBuilderList(self) 862 def _sent(slist): 863 if not slist: 864 return 865 dl = [] 866 for name, remote in slist.items(): 867 # use get() since we might have changed our mind since then. 868 # we're checking on the builder in addition to the 869 # slavebuilders out of a bit of paranoia. 870 b = self.botmaster.builders.get(name) 871 sb = self.slavebuilders.get(name) 872 if b and sb: 873 d1 = sb.attached(self, remote, self.slave_commands) 874 dl.append(d1) 875 return defer.DeferredList(dl)

876 def _set_failed(why): 877 log.msg("BuildSlave.sendBuilderList (%s) failed" % self) 878 log.err(why) 879 # TODO: hang up on them?, without setBuilderList we can't use 880 # them 881 if self.substantiation_deferred: 882 d = self.substantiation_deferred 883 self.substantiation_deferred = None 884 self.substantiation_build = None 885 d.errback(why) 886 if self.missing_timer: 887 self.missing_timer.cancel() 888 self.missing_timer = None 889 # TODO: maybe log? send an email? 890 return why 891 d.addCallbacks(_sent, _set_failed) 892 def _substantiated(res): 893 log.msg("Slave %s substantiated \o/" % self.slavename) 894 self.substantiated = True 895 if not self.substantiation_deferred: 896 log.msg("No substantiation deferred for %s" % self.slavename) 897 if self.substantiation_deferred: 898 log.msg("Firing %s substantiation deferred with success" % self.slavename) 899 d = self.substantiation_deferred 900 self.substantiation_deferred = None 901 self.substantiation_build = None 902 d.callback(True) 903 # note that the missing_timer is already handled within 904 # ``attached`` 905 if not self.building: 906 self._setBuildWaitTimer() 907 d.addCallback(_substantiated) 908 return d 909

Source Code for Module buildbot.buildslave