buildbot.process.builder

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 17 import random, weakref 18 from zope.interface import implements 19 from twisted.python import log 20 from twisted.python.failure import Failure 21 from twisted.spread import pb 22 from twisted.application import service, internet 23 from twisted.internet import defer 24 25 from buildbot import interfaces, util 26 from buildbot.status.progress import Expectations 27 from buildbot.status.builder import RETRY 28 from buildbot.process.properties import Properties 29 from buildbot.util.eventual import eventually 30 31 (ATTACHING, # slave attached, still checking hostinfo/etc 32 IDLE, # idle, available for use 33 PINGING, # build about to start, making sure it is still alive 34 BUILDING, # build is running 35 LATENT, # latent slave is not substantiated; similar to idle 36 SUBSTANTIATING, 37 ) = range(6) 38 39

40 -class AbstractSlaveBuilder(pb.Referenceable):

41 """I am the master-side representative for one of the 42 L{buildbot.slave.bot.SlaveBuilder} objects that lives in a remote 43 buildbot. When a remote builder connects, I query it for command versions 44 and then make it available to any Builds that are ready to run. """ 45

46 - def __init__(self):

47 self.ping_watchers = [] 48 self.state = None # set in subclass 49 self.remote = None 50 self.slave = None 51 self.builder_name = None 52 self.locks = None

53

54 - def __repr__(self):

55 r = ["<", self.__class__.__name__] 56 if self.builder_name: 57 r.extend([" builder=", repr(self.builder_name)]) 58 if self.slave: 59 r.extend([" slave=", repr(self.slave.slavename)]) 60 r.append(">") 61 return ''.join(r)

62

63 - def setBuilder(self, b):

64 self.builder = b 65 self.builder_name = b.name

66

67 - def getSlaveCommandVersion(self, command, oldversion=None):

68 if self.remoteCommands is None: 69 # the slave is 0.5.0 or earlier 70 return oldversion 71 return self.remoteCommands.get(command)

72

73 - def isAvailable(self):

74 # if this SlaveBuilder is busy, then it's definitely not available 75 if self.isBusy(): 76 return False 77 78 # otherwise, check in with the BuildSlave 79 if self.slave: 80 return self.slave.canStartBuild() 81 82 # no slave? not very available. 83 return False

84

85 - def isBusy(self):

86 return self.state not in (IDLE, LATENT)

87

88 - def buildStarted(self):

89 self.state = BUILDING

90

91 - def buildFinished(self):

92 self.state = IDLE 93 self.builder.triggerNewBuildCheck()

94

95 - def attached(self, slave, remote, commands):

96 """ 97 @type slave: L{buildbot.buildslave.BuildSlave} 98 @param slave: the BuildSlave that represents the buildslave as a 99 whole 100 @type remote: L{twisted.spread.pb.RemoteReference} 101 @param remote: a reference to the L{buildbot.slave.bot.SlaveBuilder} 102 @type commands: dict: string -> string, or None 103 @param commands: provides the slave's version of each RemoteCommand 104 """ 105 self.state = ATTACHING 106 self.remote = remote 107 self.remoteCommands = commands # maps command name to version 108 if self.slave is None: 109 self.slave = slave 110 self.slave.addSlaveBuilder(self) 111 else: 112 assert self.slave == slave 113 log.msg("Buildslave %s attached to %s" % (slave.slavename, 114 self.builder_name)) 115 def _attachFailure(why, where): 116 log.msg(where) 117 log.err(why) 118 return why

119 120 d = defer.succeed(None) 121 def doSetMaster(res): 122 d = self.remote.callRemote("setMaster", self) 123 #d.addErrback(_attachFailure, "Builder.setMaster") 124 return d

125 d.addCallback(doSetMaster) 126 def doPrint(res): 127 d = self.remote.callRemote("print", "attached") 128 #d.addErrback(_attachFailure, "Builder.print 'attached'") 129 return d 130 d.addCallback(doPrint) 131 def setIdle(res): 132 self.state = IDLE 133 return self 134 d.addCallback(setIdle) 135 return d 136

137 - def prepare(self, builder_status):

138 if not self.slave.acquireLocks(): 139 return defer.succeed(False) 140 return defer.succeed(True)

141

142 - def ping(self, status=None):

143 """Ping the slave to make sure it is still there. Returns a Deferred 144 that fires with True if it is. 145 146 @param status: if you point this at a BuilderStatus, a 'pinging' 147 event will be pushed. 148 """ 149 oldstate = self.state 150 self.state = PINGING 151 newping = not self.ping_watchers 152 d = defer.Deferred() 153 self.ping_watchers.append(d) 154 if newping: 155 if status: 156 event = status.addEvent(["pinging"]) 157 d2 = defer.Deferred() 158 d2.addCallback(self._pong_status, event) 159 self.ping_watchers.insert(0, d2) 160 # I think it will make the tests run smoother if the status 161 # is updated before the ping completes 162 Ping().ping(self.remote).addCallback(self._pong) 163 164 def reset_state(res): 165 if self.state == PINGING: 166 self.state = oldstate 167 return res

168 d.addCallback(reset_state) 169 return d 170

171 - def _pong(self, res):

172 watchers, self.ping_watchers = self.ping_watchers, [] 173 for d in watchers: 174 d.callback(res)

175

176 - def _pong_status(self, res, event):

177 if res: 178 event.text = ["ping", "success"] 179 else: 180 event.text = ["ping", "failed"] 181 event.finish()

182

183 - def detached(self):

184 log.msg("Buildslave %s detached from %s" % (self.slave.slavename, 185 self.builder_name)) 186 if self.slave: 187 self.slave.removeSlaveBuilder(self) 188 self.slave = None 189 self.remote = None 190 self.remoteCommands = None

191 192

193 -class Ping:

194 running = False 195

196 - def ping(self, remote):

197 assert not self.running 198 if not remote: 199 # clearly the ping must fail 200 return defer.succeed(False) 201 self.running = True 202 log.msg("sending ping") 203 self.d = defer.Deferred() 204 # TODO: add a distinct 'ping' command on the slave.. using 'print' 205 # for this purpose is kind of silly. 206 remote.callRemote("print", "ping").addCallbacks(self._pong, 207 self._ping_failed, 208 errbackArgs=(remote,)) 209 return self.d

210

211 - def _pong(self, res):

212 log.msg("ping finished: success") 213 self.d.callback(True)

214

215 - def _ping_failed(self, res, remote):

216 log.msg("ping finished: failure") 217 # the slave has some sort of internal error, disconnect them. If we 218 # don't, we'll requeue a build and ping them again right away, 219 # creating a nasty loop. 220 remote.broker.transport.loseConnection() 221 # TODO: except, if they actually did manage to get this far, they'll 222 # probably reconnect right away, and we'll do this game again. Maybe 223 # it would be better to leave them in the PINGING state. 224 self.d.callback(False)

225 226

227 -class SlaveBuilder(AbstractSlaveBuilder):

228

229 - def __init__(self):

230 AbstractSlaveBuilder.__init__(self) 231 self.state = ATTACHING

232

233 - def detached(self):

234 AbstractSlaveBuilder.detached(self) 235 if self.slave: 236 self.slave.removeSlaveBuilder(self) 237 self.slave = None 238 self.state = ATTACHING

239

240 - def buildFinished(self):

241 # Call the slave's buildFinished if we can; the slave may be waiting 242 # to do a graceful shutdown and needs to know when it's idle. 243 # After, we check to see if we can start other builds. 244 self.state = IDLE 245 if self.slave: 246 d = self.slave.buildFinished(self) 247 d.addCallback(lambda x: self.builder.triggerNewBuildCheck()) 248 else: 249 self.builder.triggerNewBuildCheck()

250 251

252 -class LatentSlaveBuilder(AbstractSlaveBuilder):

253 - def __init__(self, slave, builder):

254 AbstractSlaveBuilder.__init__(self) 255 self.slave = slave 256 self.state = LATENT 257 self.setBuilder(builder) 258 self.slave.addSlaveBuilder(self) 259 log.msg("Latent buildslave %s attached to %s" % (slave.slavename, 260 self.builder_name))

261

262 - def prepare(self, builder_status):

263 # If we can't lock, then don't bother trying to substantiate 264 if not self.slave.acquireLocks(): 265 return defer.succeed(False) 266 267 log.msg("substantiating slave %s" % (self,)) 268 d = self.substantiate() 269 def substantiation_failed(f): 270 builder_status.addPointEvent(['removing', 'latent', 271 self.slave.slavename]) 272 self.slave.disconnect() 273 # TODO: should failover to a new Build 274 return f

275 def substantiation_cancelled(res): 276 # if res is False, latent slave cancelled subtantiation 277 if not res: 278 self.state = LATENT 279 return res

280 d.addCallback(substantiation_cancelled) 281 d.addErrback(substantiation_failed) 282 return d 283

284 - def substantiate(self):

285 self.state = SUBSTANTIATING 286 d = self.slave.substantiate(self) 287 if not self.slave.substantiated: 288 event = self.builder.builder_status.addEvent( 289 ["substantiating"]) 290 def substantiated(res): 291 msg = ["substantiate", "success"] 292 if isinstance(res, basestring): 293 msg.append(res) 294 elif isinstance(res, (tuple, list)): 295 msg.extend(res) 296 event.text = msg 297 event.finish() 298 return res

299 def substantiation_failed(res): 300 event.text = ["substantiate", "failed"] 301 # TODO add log of traceback to event 302 event.finish() 303 return res 304 d.addCallbacks(substantiated, substantiation_failed) 305 return d 306

307 - def detached(self):

308 AbstractSlaveBuilder.detached(self) 309 self.state = LATENT

310

311 - def buildStarted(self):

312 AbstractSlaveBuilder.buildStarted(self) 313 self.slave.buildStarted(self)

314

315 - def buildFinished(self):

316 AbstractSlaveBuilder.buildFinished(self) 317 self.slave.buildFinished(self)

318

319 - def _attachFailure(self, why, where):

320 self.state = LATENT 321 return AbstractSlaveBuilder._attachFailure(self, why, where)

322

323 - def ping(self, status=None):

324 if not self.slave.substantiated: 325 if status: 326 status.addEvent(["ping", "latent"]).finish() 327 return defer.succeed(True) 328 return AbstractSlaveBuilder.ping(self, status)

329 330

331 -class Builder(pb.Referenceable, service.MultiService):

332 """I manage all Builds of a given type. 333 334 Each Builder is created by an entry in the config file (the c['builders'] 335 list), with a number of parameters. 336 337 One of these parameters is the L{buildbot.process.factory.BuildFactory} 338 object that is associated with this Builder. The factory is responsible 339 for creating new L{Build<buildbot.process.base.Build>} objects. Each 340 Build object defines when and how the build is performed, so a new 341 Factory or Builder should be defined to control this behavior. 342 343 The Builder holds on to a number of L{base.BuildRequest} objects in a 344 list named C{.buildable}. Incoming BuildRequest objects will be added to 345 this list, or (if possible) merged into an existing request. When a slave 346 becomes available, I will use my C{BuildFactory} to turn the request into 347 a new C{Build} object. The C{BuildRequest} is forgotten, the C{Build} 348 goes into C{.building} while it runs. Once the build finishes, I will 349 discard it. 350 351 I maintain a list of available SlaveBuilders, one for each connected 352 slave that the C{slavenames} parameter says we can use. Some of these 353 will be idle, some of them will be busy running builds for me. If there 354 are multiple slaves, I can run multiple builds at once. 355 356 I also manage forced builds, progress expectation (ETA) management, and 357 some status delivery chores. 358 359 @type buildable: list of L{buildbot.process.base.BuildRequest} 360 @ivar buildable: BuildRequests that are ready to build, but which are 361 waiting for a buildslave to be available. 362 363 @type building: list of L{buildbot.process.base.Build} 364 @ivar building: Builds that are actively running 365 366 @type slaves: list of L{buildbot.buildslave.BuildSlave} objects 367 @ivar slaves: the slaves currently available for building 368 """ 369 370 expectations = None # this is created the first time we get a good build 371 CHOOSE_SLAVES_RANDOMLY = True # disabled for determinism during tests 372

373 - def __init__(self, setup, builder_status):

374 """ 375 @type setup: dict 376 @param setup: builder setup data, as stored in 377 BuildmasterConfig['builders']. Contains name, 378 slavename(s), builddir, slavebuilddir, factory, locks. 379 @type builder_status: L{buildbot.status.builder.BuilderStatus} 380 """ 381 service.MultiService.__init__(self) 382 self.name = setup['name'] 383 self.slavenames = [] 384 if setup.has_key('slavename'): 385 self.slavenames.append(setup['slavename']) 386 if setup.has_key('slavenames'): 387 self.slavenames.extend(setup['slavenames']) 388 self.builddir = setup['builddir'] 389 self.slavebuilddir = setup['slavebuilddir'] 390 self.buildFactory = setup['factory'] 391 self.nextSlave = setup.get('nextSlave') 392 if self.nextSlave is not None and not callable(self.nextSlave): 393 raise ValueError("nextSlave must be callable") 394 self.locks = setup.get("locks", []) 395 self.env = setup.get('env', {}) 396 assert isinstance(self.env, dict) 397 if setup.has_key('periodicBuildTime'): 398 raise ValueError("periodicBuildTime can no longer be defined as" 399 " part of the Builder: use scheduler.Periodic" 400 " instead") 401 self.nextBuild = setup.get('nextBuild') 402 if self.nextBuild is not None and not callable(self.nextBuild): 403 raise ValueError("nextBuild must be callable") 404 self.buildHorizon = setup.get('buildHorizon') 405 self.logHorizon = setup.get('logHorizon') 406 self.eventHorizon = setup.get('eventHorizon') 407 self.mergeRequests = setup.get('mergeRequests', True) 408 self.properties = setup.get('properties', {}) 409 self.category = setup.get('category', None) 410 411 # build/wannabuild slots: Build objects move along this sequence 412 self.building = [] 413 # old_building holds active builds that were stolen from a predecessor 414 self.old_building = weakref.WeakKeyDictionary() 415 416 # buildslaves which have connected but which are not yet available. 417 # These are always in the ATTACHING state. 418 self.attaching_slaves = [] 419 420 # buildslaves at our disposal. Each SlaveBuilder instance has a 421 # .state that is IDLE, PINGING, or BUILDING. "PINGING" is used when a 422 # Build is about to start, to make sure that they're still alive. 423 self.slaves = [] 424 425 self.builder_status = builder_status 426 self.builder_status.setSlavenames(self.slavenames) 427 self.builder_status.buildHorizon = self.buildHorizon 428 self.builder_status.logHorizon = self.logHorizon 429 self.builder_status.eventHorizon = self.eventHorizon 430 t = internet.TimerService(10*60, self.reclaimAllBuilds) 431 t.setServiceParent(self) 432 433 # for testing, to help synchronize tests 434 self.watchers = {'attach': [], 'detach': [], 'detach_all': [], 435 'idle': []} 436 self.run_count = 0

437

438 - def setBotmaster(self, botmaster):

439 self.botmaster = botmaster 440 self.db = botmaster.db 441 self.master_name = botmaster.master_name 442 self.master_incarnation = botmaster.master_incarnation

443

444 - def compareToSetup(self, setup):

445 diffs = [] 446 setup_slavenames = [] 447 if setup.has_key('slavename'): 448 setup_slavenames.append(setup['slavename']) 449 setup_slavenames.extend(setup.get('slavenames', [])) 450 if setup_slavenames != self.slavenames: 451 diffs.append('slavenames changed from %s to %s' \ 452 % (self.slavenames, setup_slavenames)) 453 if setup['builddir'] != self.builddir: 454 diffs.append('builddir changed from %s to %s' \ 455 % (self.builddir, setup['builddir'])) 456 if setup['slavebuilddir'] != self.slavebuilddir: 457 diffs.append('slavebuilddir changed from %s to %s' \ 458 % (self.slavebuilddir, setup['slavebuilddir'])) 459 if setup['factory'] != self.buildFactory: # compare objects 460 diffs.append('factory changed') 461 if setup.get('locks', []) != self.locks: 462 diffs.append('locks changed from %s to %s' % (self.locks, setup.get('locks'))) 463 if setup.get('env', {}) != self.env: 464 diffs.append('env changed from %s to %s' % (self.env, setup.get('env', {}))) 465 if setup.get('nextSlave') != self.nextSlave: 466 diffs.append('nextSlave changed from %s to %s' % (self.nextSlave, setup.get('nextSlave'))) 467 if setup.get('nextBuild') != self.nextBuild: 468 diffs.append('nextBuild changed from %s to %s' % (self.nextBuild, setup.get('nextBuild'))) 469 if setup['buildHorizon'] != self.buildHorizon: 470 diffs.append('buildHorizon changed from %s to %s' % (self.buildHorizon, setup['buildHorizon'])) 471 if setup['logHorizon'] != self.logHorizon: 472 diffs.append('logHorizon changed from %s to %s' % (self.logHorizon, setup['logHorizon'])) 473 if setup['eventHorizon'] != self.eventHorizon: 474 diffs.append('eventHorizon changed from %s to %s' % (self.eventHorizon, setup['eventHorizon'])) 475 if setup['category'] != self.category: 476 diffs.append('category changed from %r to %r' % (self.category, setup['category'])) 477 478 return diffs

479

480 - def __repr__(self):

481 return "<Builder '%r' at %d>" % (self.name, id(self))

482

483 - def triggerNewBuildCheck(self):

484 self.botmaster.triggerNewBuildCheck()

485

486 - def run(self):

487 """Check for work to be done. This should be called any time I might 488 be able to start a job: 489 490 - when the Builder is first created 491 - when a new job has been added to the [buildrequests] DB table 492 - when a slave has connected 493 494 If I have both an available slave and the database contains a 495 BuildRequest that I can handle, I will claim the BuildRequest and 496 start the build. When the build finishes, I will retire the 497 BuildRequest. 498 """ 499 # overall plan: 500 # move .expectations to DB 501 502 # if we're not running, we may still be called from leftovers from 503 # a run of the loop, so just ignore the call. 504 if not self.running: 505 return 506 507 self.run_count += 1 508 509 available_slaves = [sb for sb in self.slaves if sb.isAvailable()] 510 if not available_slaves: 511 self.updateBigStatus() 512 return 513 d = self.db.runInteraction(self._claim_buildreqs, available_slaves) 514 d.addCallback(self._start_builds) 515 return d

516 517 # slave-managers must refresh their claim on a build at least once an 518 # hour, less any inter-manager clock skew 519 RECLAIM_INTERVAL = 1*3600 520

521 - def _claim_buildreqs(self, t, available_slaves):

522 # return a dict mapping slave -> (brid,ssid) 523 now = util.now() 524 old = now - self.RECLAIM_INTERVAL 525 requests = self.db.get_unclaimed_buildrequests(self.name, old, 526 self.master_name, 527 self.master_incarnation, 528 t) 529 530 assignments = {} 531 while requests and available_slaves: 532 sb = self._choose_slave(available_slaves) 533 if not sb: 534 log.msg("%s: want to start build, but we don't have a remote" 535 % self) 536 break 537 available_slaves.remove(sb) 538 breq = self._choose_build(requests) 539 if not breq: 540 log.msg("%s: went to start build, but nextBuild said not to" 541 % self) 542 break 543 requests.remove(breq) 544 merged_requests = [breq] 545 for other_breq in requests[:]: 546 if (self.mergeRequests and 547 self.botmaster.shouldMergeRequests(self, breq, other_breq) 548 ): 549 requests.remove(other_breq) 550 merged_requests.append(other_breq) 551 assignments[sb] = merged_requests 552 brids = [br.id for br in merged_requests] 553 self.db.claim_buildrequests(now, self.master_name, 554 self.master_incarnation, brids, t) 555 return assignments

556

557 - def _choose_slave(self, available_slaves):

558 # note: this might return None if the nextSlave() function decided to 559 # not give us anything 560 if self.nextSlave: 561 try: 562 return self.nextSlave(self, available_slaves) 563 except: 564 log.msg("Exception choosing next slave") 565 log.err(Failure()) 566 return None 567 if self.CHOOSE_SLAVES_RANDOMLY: 568 return random.choice(available_slaves) 569 return available_slaves[0]

570

571 - def _choose_build(self, buildable):

572 if self.nextBuild: 573 try: 574 return self.nextBuild(self, buildable) 575 except: 576 log.msg("Exception choosing next build") 577 log.err(Failure()) 578 return None 579 return buildable[0]

580

581 - def _start_builds(self, assignments):

582 # because _claim_buildreqs runs in a separate thread, we might have 583 # lost a slave by this point. We treat that case the same as if we 584 # lose the slave right after the build starts: the initial ping 585 # fails. 586 for (sb, requests) in assignments.items(): 587 build = self.buildFactory.newBuild(requests) 588 build.setBuilder(self) 589 build.setLocks(self.locks) 590 if len(self.env) > 0: 591 build.setSlaveEnvironment(self.env) 592 self.startBuild(build, sb) 593 self.updateBigStatus()

594 595

596 - def getBuildable(self, limit=None):

597 return self.db.runInteractionNow(self._getBuildable, limit)

598 - def _getBuildable(self, t, limit):

599 now = util.now() 600 old = now - self.RECLAIM_INTERVAL 601 return self.db.get_unclaimed_buildrequests(self.name, old, 602 self.master_name, 603 self.master_incarnation, 604 t, 605 limit)

606

607 - def getOldestRequestTime(self):

608 """Returns the timestamp of the oldest build request for this builder. 609 610 If there are no build requests, None is returned.""" 611 buildable = self.getBuildable(1) 612 if buildable: 613 # TODO: this is sorted by priority first, not strictly reqtime 614 return buildable[0].getSubmitTime() 615 return None

616

617 - def cancelBuildRequest(self, brid):

618 return self.db.cancel_buildrequests([brid])

619

620 - def consumeTheSoulOfYourPredecessor(self, old):

621 """Suck the brain out of an old Builder. 622 623 This takes all the runtime state from an existing Builder and moves 624 it into ourselves. This is used when a Builder is changed in the 625 master.cfg file: the new Builder has a different factory, but we want 626 all the builds that were queued for the old one to get processed by 627 the new one. Any builds which are already running will keep running. 628 The new Builder will get as many of the old SlaveBuilder objects as 629 it wants.""" 630 631 log.msg("consumeTheSoulOfYourPredecessor: %s feeding upon %s" % 632 (self, old)) 633 # all pending builds are stored in the DB, so we don't have to do 634 # anything to claim them. The old builder will be stopService'd, 635 # which should make sure they don't start any new work 636 637 # this is kind of silly, but the builder status doesn't get updated 638 # when the config changes, yet it stores the category. So: 639 self.builder_status.category = self.category 640 641 # old.building (i.e. builds which are still running) is not migrated 642 # directly: it keeps track of builds which were in progress in the 643 # old Builder. When those builds finish, the old Builder will be 644 # notified, not us. However, since the old SlaveBuilder will point to 645 # us, it is our maybeStartBuild() that will be triggered. 646 if old.building: 647 self.builder_status.setBigState("building") 648 # however, we do grab a weakref to the active builds, so that our 649 # BuilderControl can see them and stop them. We use a weakref because 650 # we aren't the one to get notified, so there isn't a convenient 651 # place to remove it from self.building . 652 for b in old.building: 653 self.old_building[b] = None 654 for b in old.old_building: 655 self.old_building[b] = None 656 657 # Our set of slavenames may be different. Steal any of the old 658 # buildslaves that we want to keep using. 659 for sb in old.slaves[:]: 660 if sb.slave.slavename in self.slavenames: 661 log.msg(" stealing buildslave %s" % sb) 662 self.slaves.append(sb) 663 old.slaves.remove(sb) 664 sb.setBuilder(self) 665 666 # old.attaching_slaves: 667 # these SlaveBuilders are waiting on a sequence of calls: 668 # remote.setMaster and remote.print . When these two complete, 669 # old._attached will be fired, which will add a 'connect' event to 670 # the builder_status and try to start a build. However, we've pulled 671 # everything out of the old builder's queue, so it will have no work 672 # to do. The outstanding remote.setMaster/print call will be holding 673 # the last reference to the old builder, so it will disappear just 674 # after that response comes back. 675 # 676 # The BotMaster will ask the slave to re-set their list of Builders 677 # shortly after this function returns, which will cause our 678 # attached() method to be fired with a bunch of references to remote 679 # SlaveBuilders, some of which we already have (by stealing them 680 # from the old Builder), some of which will be new. The new ones 681 # will be re-attached. 682 683 # Therefore, we don't need to do anything about old.attaching_slaves 684 685 return # all done

686

687 - def reclaimAllBuilds(self):

688 try: 689 now = util.now() 690 brids = set() 691 for b in self.building: 692 brids.update([br.id for br in b.requests]) 693 for b in self.old_building: 694 brids.update([br.id for br in b.requests]) 695 self.db.claim_buildrequests(now, self.master_name, 696 self.master_incarnation, brids) 697 except: 698 log.msg("Error in reclaimAllBuilds") 699 log.err()

700

701 - def getBuild(self, number):

702 for b in self.building: 703 if b.build_status and b.build_status.number == number: 704 return b 705 for b in self.old_building.keys(): 706 if b.build_status and b.build_status.number == number: 707 return b 708 return None

709

710 - def fireTestEvent(self, name, fire_with=None):

711 if fire_with is None: 712 fire_with = self 713 watchers = self.watchers[name] 714 self.watchers[name] = [] 715 for w in watchers: 716 eventually(w.callback, fire_with)

717

718 - def addLatentSlave(self, slave):

719 assert interfaces.ILatentBuildSlave.providedBy(slave) 720 for s in self.slaves: 721 if s == slave: 722 break 723 else: 724 sb = LatentSlaveBuilder(slave, self) 725 self.builder_status.addPointEvent( 726 ['added', 'latent', slave.slavename]) 727 self.slaves.append(sb) 728 self.triggerNewBuildCheck()

729

730 - def attached(self, slave, remote, commands):

731 """This is invoked by the BuildSlave when the self.slavename bot 732 registers their builder. 733 734 @type slave: L{buildbot.buildslave.BuildSlave} 735 @param slave: the BuildSlave that represents the buildslave as a whole 736 @type remote: L{twisted.spread.pb.RemoteReference} 737 @param remote: a reference to the L{buildbot.slave.bot.SlaveBuilder} 738 @type commands: dict: string -> string, or None 739 @param commands: provides the slave's version of each RemoteCommand 740 741 @rtype: L{twisted.internet.defer.Deferred} 742 @return: a Deferred that fires (with 'self') when the slave-side 743 builder is fully attached and ready to accept commands. 744 """ 745 for s in self.attaching_slaves + self.slaves: 746 if s.slave == slave: 747 # already attached to them. This is fairly common, since 748 # attached() gets called each time we receive the builder 749 # list from the slave, and we ask for it each time we add or 750 # remove a builder. So if the slave is hosting builders 751 # A,B,C, and the config file changes A, we'll remove A and 752 # re-add it, triggering two builder-list requests, getting 753 # two redundant calls to attached() for B, and another two 754 # for C. 755 # 756 # Therefore, when we see that we're already attached, we can 757 # just ignore it. TODO: build a diagram of the state 758 # transitions here, I'm concerned about sb.attached() failing 759 # and leaving sb.state stuck at 'ATTACHING', and about 760 # the detached() message arriving while there's some 761 # transition pending such that the response to the transition 762 # re-vivifies sb 763 return defer.succeed(self) 764 765 sb = SlaveBuilder() 766 sb.setBuilder(self) 767 self.attaching_slaves.append(sb) 768 d = sb.attached(slave, remote, commands) 769 d.addCallback(self._attached) 770 d.addErrback(self._not_attached, slave) 771 return d

772

773 - def _attached(self, sb):

774 # TODO: make this .addSlaveEvent(slave.slavename, ['connect']) ? 775 self.builder_status.addPointEvent(['connect', sb.slave.slavename]) 776 self.attaching_slaves.remove(sb) 777 self.slaves.append(sb) 778 779 self.fireTestEvent('attach') 780 return self

781

782 - def _not_attached(self, why, slave):

783 # already log.err'ed by SlaveBuilder._attachFailure 784 # TODO: make this .addSlaveEvent? 785 # TODO: remove from self.slaves (except that detached() should get 786 # run first, right?) 787 print why 788 self.builder_status.addPointEvent(['failed', 'connect', 789 slave.slavename]) 790 # TODO: add an HTMLLogFile of the exception 791 self.fireTestEvent('attach', why)

792

793 - def detached(self, slave):

794 """This is called when the connection to the bot is lost.""" 795 for sb in self.attaching_slaves + self.slaves: 796 if sb.slave == slave: 797 break 798 else: 799 log.msg("WEIRD: Builder.detached(%s) (%s)" 800 " not in attaching_slaves(%s)" 801 " or slaves(%s)" % (slave, slave.slavename, 802 self.attaching_slaves, 803 self.slaves)) 804 return 805 if sb.state == BUILDING: 806 # the Build's .lostRemote method (invoked by a notifyOnDisconnect 807 # handler) will cause the Build to be stopped, probably right 808 # after the notifyOnDisconnect that invoked us finishes running. 809 810 # TODO: should failover to a new Build 811 #self.retryBuild(sb.build) 812 pass 813 814 if sb in self.attaching_slaves: 815 self.attaching_slaves.remove(sb) 816 if sb in self.slaves: 817 self.slaves.remove(sb) 818 819 # TODO: make this .addSlaveEvent? 820 self.builder_status.addPointEvent(['disconnect', slave.slavename]) 821 sb.detached() # inform the SlaveBuilder that their slave went away 822 self.updateBigStatus() 823 self.fireTestEvent('detach') 824 if not self.slaves: 825 self.fireTestEvent('detach_all')

826

827 - def updateBigStatus(self):

828 if not self.slaves: 829 self.builder_status.setBigState("offline") 830 elif self.building: 831 self.builder_status.setBigState("building") 832 else: 833 self.builder_status.setBigState("idle") 834 self.fireTestEvent('idle')

835

836 - def startBuild(self, build, sb):

837 """Start a build on the given slave. 838 @param build: the L{base.Build} to start 839 @param sb: the L{SlaveBuilder} which will host this build 840 841 @return: a Deferred which fires with a 842 L{buildbot.interfaces.IBuildControl} that can be used to stop the 843 Build, or to access a L{buildbot.interfaces.IBuildStatus} which will 844 watch the Build as it runs. """ 845 846 self.building.append(build) 847 self.updateBigStatus() 848 log.msg("starting build %s using slave %s" % (build, sb)) 849 d = sb.prepare(self.builder_status) 850 851 def _prepared(ready): 852 # If prepare returns True then it is ready and we start a build 853 # If it returns false then we don't start a new build. 854 d = defer.succeed(ready) 855 856 if not ready: 857 #FIXME: We should perhaps trigger a check to see if there is 858 # any other way to schedule the work 859 log.msg("slave %s can't build %s after all" % (build, sb)) 860 861 # release the slave. This will queue a call to maybeStartBuild, which 862 # will fire after other notifyOnDisconnect handlers have marked the 863 # slave as disconnected (so we don't try to use it again). 864 # sb.buildFinished() 865 866 log.msg("re-queueing the BuildRequest %s" % build) 867 self.building.remove(build) 868 self._resubmit_buildreqs(build).addErrback(log.err) 869 870 sb.slave.releaseLocks() 871 self.triggerNewBuildCheck() 872 873 return d 874 875 def _ping(ign): 876 # ping the slave to make sure they're still there. If they've 877 # fallen off the map (due to a NAT timeout or something), this 878 # will fail in a couple of minutes, depending upon the TCP 879 # timeout. 880 # 881 # TODO: This can unnecessarily suspend the starting of a build, in 882 # situations where the slave is live but is pushing lots of data to 883 # us in a build. 884 log.msg("starting build %s.. pinging the slave %s" % (build, sb)) 885 return sb.ping()

886 d.addCallback(_ping) 887 d.addCallback(self._startBuild_1, build, sb) 888 889 return d

890 891 d.addCallback(_prepared) 892 return d 893

894 - def _startBuild_1(self, res, build, sb):

895 if not res: 896 return self._startBuildFailed("slave ping failed", build, sb) 897 # The buildslave is ready to go. sb.buildStarted() sets its state to 898 # BUILDING (so we won't try to use it for any other builds). This 899 # gets set back to IDLE by the Build itself when it finishes. 900 sb.buildStarted() 901 d = sb.remote.callRemote("startBuild") 902 d.addCallbacks(self._startBuild_2, self._startBuildFailed, 903 callbackArgs=(build,sb), errbackArgs=(build,sb)) 904 return d

905

906 - def _startBuild_2(self, res, build, sb):

907 # create the BuildStatus object that goes with the Build 908 bs = self.builder_status.newBuild() 909 910 # start the build. This will first set up the steps, then tell the 911 # BuildStatus that it has started, which will announce it to the 912 # world (through our BuilderStatus object, which is its parent). 913 # Finally it will start the actual build process. 914 bids = [self.db.build_started(req.id, bs.number) for req in build.requests] 915 d = build.startBuild(bs, self.expectations, sb) 916 d.addCallback(self.buildFinished, sb, bids) 917 # this shouldn't happen. if it does, the slave will be wedged 918 d.addErrback(log.err) 919 return build # this is the IBuildControl

920

921 - def _startBuildFailed(self, why, build, sb):

922 # put the build back on the buildable list 923 log.msg("I tried to tell the slave that the build %s started, but " 924 "remote_startBuild failed: %s" % (build, why)) 925 # release the slave. This will queue a call to maybeStartBuild, which 926 # will fire after other notifyOnDisconnect handlers have marked the 927 # slave as disconnected (so we don't try to use it again). 928 sb.buildFinished() 929 930 log.msg("re-queueing the BuildRequest") 931 self.building.remove(build) 932 self._resubmit_buildreqs(build).addErrback(log.err)

933

934 - def setupProperties(self, props):

935 props.setProperty("buildername", self.name, "Builder") 936 if len(self.properties) > 0: 937 for propertyname in self.properties: 938 props.setProperty(propertyname, self.properties[propertyname], "Builder")

939

940 - def buildFinished(self, build, sb, bids):

941 """This is called when the Build has finished (either success or 942 failure). Any exceptions during the build are reported with 943 results=FAILURE, not with an errback.""" 944 945 # by the time we get here, the Build has already released the slave 946 # (which queues a call to maybeStartBuild) 947 948 self.db.builds_finished(bids) 949 950 results = build.build_status.getResults() 951 self.building.remove(build) 952 if results == RETRY: 953 self._resubmit_buildreqs(build).addErrback(log.err) # returns Deferred 954 else: 955 brids = [br.id for br in build.requests] 956 self.db.retire_buildrequests(brids, results) 957 958 if sb.slave: 959 sb.slave.releaseLocks() 960 961 self.triggerNewBuildCheck()

962

963 - def _resubmit_buildreqs(self, build):

964 brids = [br.id for br in build.requests] 965 return self.db.resubmit_buildrequests(brids)

966

967 - def setExpectations(self, progress):

968 """Mark the build as successful and update expectations for the next 969 build. Only call this when the build did not fail in any way that 970 would invalidate the time expectations generated by it. (if the 971 compile failed and thus terminated early, we can't use the last 972 build to predict how long the next one will take). 973 """ 974 if self.expectations: 975 self.expectations.update(progress) 976 else: 977 # the first time we get a good build, create our Expectations 978 # based upon its results 979 self.expectations = Expectations(progress) 980 log.msg("new expectations: %s seconds" % \ 981 self.expectations.expectedBuildTime())

982

983 - def shutdownSlave(self):

984 if self.remote: 985 self.remote.callRemote("shutdown")

986 987

988 -class BuilderControl:

989 implements(interfaces.IBuilderControl) 990

991 - def __init__(self, builder, parent):

992 self.original = builder 993 self.parent = parent # the IControl object

994

995 - def submitBuildRequest(self, ss, reason, props=None, now=False):

996 bss = self.parent.submitBuildSet([self.original.name], ss, reason, 997 props, now) 998 brs = bss.getBuildRequests()[0] 999 return brs

1000

1001 - def rebuildBuild(self, bs, reason="<rebuild, no reason given>", extraProperties=None):

1002 if not bs.isFinished(): 1003 return 1004 1005 ss = bs.getSourceStamp(absolute=True) 1006 # Make a copy so as not to modify the original build. 1007 properties = Properties() 1008 # Don't include runtime-set properties in a rebuild request 1009 properties.updateFromPropertiesNoRuntime(bs.getProperties()) 1010 if extraProperties is None: 1011 properties.updateFromProperties(extraProperties) 1012 self.submitBuildRequest(ss, reason, props=properties)

1013

1014 - def getPendingBuilds(self):

1015 # return IBuildRequestControl objects 1016 retval = [] 1017 for r in self.original.getBuildable(): 1018 retval.append(BuildRequestControl(self.original, r)) 1019 1020 return retval

1021

1022 - def getBuild(self, number):

1023 return self.original.getBuild(number)

1024

1025 - def ping(self):

1026 if not self.original.slaves: 1027 self.original.builder_status.addPointEvent(["ping", "no slave"]) 1028 return defer.succeed(False) # interfaces.NoSlaveError 1029 dl = [] 1030 for s in self.original.slaves: 1031 dl.append(s.ping(self.original.builder_status)) 1032 d = defer.DeferredList(dl) 1033 d.addCallback(self._gatherPingResults) 1034 return d

1035

1036 - def _gatherPingResults(self, res):

1037 for ignored,success in res: 1038 if not success: 1039 return False 1040 return True

1041

1042 -class BuildRequestControl:

1043 implements(interfaces.IBuildRequestControl) 1044

1045 - def __init__(self, builder, request):

1046 self.original_builder = builder 1047 self.original_request = request 1048 self.brid = request.id

1049 1052

1053 - def unsubscribe(self, observer):

1054 raise NotImplementedError

1055

1056 - def cancel(self):

1057 self.original_builder.cancelBuildRequest(self.brid)

1058

Source Code for Module buildbot.process.builder