buildbot.process.builder

1 2 import random, weakref 3 from zope.interface import implements 4 from twisted.python import log 5 from twisted.python.failure import Failure 6 from twisted.spread import pb 7 from twisted.application import service, internet 8 from twisted.internet import defer 9 10 from buildbot import interfaces, util 11 from buildbot.status.progress import Expectations 12 from buildbot.status.builder import RETRY 13 from buildbot.process.properties import Properties 14 from buildbot.util.eventual import eventually 15 16 (ATTACHING, # slave attached, still checking hostinfo/etc 17 IDLE, # idle, available for use 18 PINGING, # build about to start, making sure it is still alive 19 BUILDING, # build is running 20 LATENT, # latent slave is not substantiated; similar to idle 21 SUBSTANTIATING, 22 ) = range(6) 23 24

25 -class AbstractSlaveBuilder(pb.Referenceable):

26 """I am the master-side representative for one of the 27 L{buildbot.slave.bot.SlaveBuilder} objects that lives in a remote 28 buildbot. When a remote builder connects, I query it for command versions 29 and then make it available to any Builds that are ready to run. """ 30

31 - def __init__(self):

32 self.ping_watchers = [] 33 self.state = None # set in subclass 34 self.remote = None 35 self.slave = None 36 self.builder_name = None

37

38 - def __repr__(self):

39 r = ["<", self.__class__.__name__] 40 if self.builder_name: 41 r.extend([" builder=", repr(self.builder_name)]) 42 if self.slave: 43 r.extend([" slave=", repr(self.slave.slavename)]) 44 r.append(">") 45 return ''.join(r)

46

47 - def setBuilder(self, b):

48 self.builder = b 49 self.builder_name = b.name

50

51 - def getSlaveCommandVersion(self, command, oldversion=None):

52 if self.remoteCommands is None: 53 # the slave is 0.5.0 or earlier 54 return oldversion 55 return self.remoteCommands.get(command)

56

57 - def isAvailable(self):

58 # if this SlaveBuilder is busy, then it's definitely not available 59 if self.isBusy(): 60 return False 61 62 # otherwise, check in with the BuildSlave 63 if self.slave: 64 return self.slave.canStartBuild() 65 66 # no slave? not very available. 67 return False

68

69 - def isBusy(self):

70 return self.state not in (IDLE, LATENT)

71

72 - def buildStarted(self):

73 self.state = BUILDING

74

75 - def buildFinished(self):

76 self.state = IDLE 77 self.builder.triggerNewBuildCheck()

78

79 - def attached(self, slave, remote, commands):

80 """ 81 @type slave: L{buildbot.buildslave.BuildSlave} 82 @param slave: the BuildSlave that represents the buildslave as a 83 whole 84 @type remote: L{twisted.spread.pb.RemoteReference} 85 @param remote: a reference to the L{buildbot.slave.bot.SlaveBuilder} 86 @type commands: dict: string -> string, or None 87 @param commands: provides the slave's version of each RemoteCommand 88 """ 89 self.state = ATTACHING 90 self.remote = remote 91 self.remoteCommands = commands # maps command name to version 92 if self.slave is None: 93 self.slave = slave 94 self.slave.addSlaveBuilder(self) 95 else: 96 assert self.slave == slave 97 log.msg("Buildslave %s attached to %s" % (slave.slavename, 98 self.builder_name)) 99 d = self.remote.callRemote("setMaster", self) 100 d.addErrback(self._attachFailure, "Builder.setMaster") 101 d.addCallback(self._attached2) 102 return d

103

104 - def _attached2(self, res):

105 d = self.remote.callRemote("print", "attached") 106 d.addErrback(self._attachFailure, "Builder.print 'attached'") 107 d.addCallback(self._attached3) 108 return d

109

110 - def _attached3(self, res):

111 # now we say they're really attached 112 self.state = IDLE 113 return self

114

115 - def _attachFailure(self, why, where):

116 assert isinstance(where, str) 117 log.msg(where) 118 log.err(why) 119 return why

120

121 - def prepare(self, builder_status):

122 return defer.succeed(None)

123

124 - def ping(self, status=None):

125 """Ping the slave to make sure it is still there. Returns a Deferred 126 that fires with True if it is. 127 128 @param status: if you point this at a BuilderStatus, a 'pinging' 129 event will be pushed. 130 """ 131 oldstate = self.state 132 self.state = PINGING 133 newping = not self.ping_watchers 134 d = defer.Deferred() 135 self.ping_watchers.append(d) 136 if newping: 137 if status: 138 event = status.addEvent(["pinging"]) 139 d2 = defer.Deferred() 140 d2.addCallback(self._pong_status, event) 141 self.ping_watchers.insert(0, d2) 142 # I think it will make the tests run smoother if the status 143 # is updated before the ping completes 144 Ping().ping(self.remote).addCallback(self._pong) 145 146 def reset_state(res): 147 if self.state == PINGING: 148 self.state = oldstate 149 return res

150 d.addCallback(reset_state) 151 return d

152

153 - def _pong(self, res):

154 watchers, self.ping_watchers = self.ping_watchers, [] 155 for d in watchers: 156 d.callback(res)

157

158 - def _pong_status(self, res, event):

159 if res: 160 event.text = ["ping", "success"] 161 else: 162 event.text = ["ping", "failed"] 163 event.finish()

164

165 - def detached(self):

166 log.msg("Buildslave %s detached from %s" % (self.slave.slavename, 167 self.builder_name)) 168 if self.slave: 169 self.slave.removeSlaveBuilder(self) 170 self.slave = None 171 self.remote = None 172 self.remoteCommands = None

173 174

175 -class Ping:

176 running = False 177

178 - def ping(self, remote):

179 assert not self.running 180 if not remote: 181 # clearly the ping must fail 182 return defer.succeed(False) 183 self.running = True 184 log.msg("sending ping") 185 self.d = defer.Deferred() 186 # TODO: add a distinct 'ping' command on the slave.. using 'print' 187 # for this purpose is kind of silly. 188 remote.callRemote("print", "ping").addCallbacks(self._pong, 189 self._ping_failed, 190 errbackArgs=(remote,)) 191 return self.d

192

193 - def _pong(self, res):

194 log.msg("ping finished: success") 195 self.d.callback(True)

196

197 - def _ping_failed(self, res, remote):

198 log.msg("ping finished: failure") 199 # the slave has some sort of internal error, disconnect them. If we 200 # don't, we'll requeue a build and ping them again right away, 201 # creating a nasty loop. 202 remote.broker.transport.loseConnection() 203 # TODO: except, if they actually did manage to get this far, they'll 204 # probably reconnect right away, and we'll do this game again. Maybe 205 # it would be better to leave them in the PINGING state. 206 self.d.callback(False)

207 208

209 -class SlaveBuilder(AbstractSlaveBuilder):

210

211 - def __init__(self):

212 AbstractSlaveBuilder.__init__(self) 213 self.state = ATTACHING

214

215 - def detached(self):

216 AbstractSlaveBuilder.detached(self) 217 if self.slave: 218 self.slave.removeSlaveBuilder(self) 219 self.slave = None 220 self.state = ATTACHING

221

222 - def buildFinished(self):

223 # Call the slave's buildFinished if we can; the slave may be waiting 224 # to do a graceful shutdown and needs to know when it's idle. 225 # After, we check to see if we can start other builds. 226 self.state = IDLE 227 if self.slave: 228 d = self.slave.buildFinished(self) 229 d.addCallback(lambda x: self.builder.triggerNewBuildCheck()) 230 else: 231 self.builder.triggerNewBuildCheck()

232 233

234 -class LatentSlaveBuilder(AbstractSlaveBuilder):

235 - def __init__(self, slave, builder):

236 AbstractSlaveBuilder.__init__(self) 237 self.slave = slave 238 self.state = LATENT 239 self.setBuilder(builder) 240 self.slave.addSlaveBuilder(self) 241 log.msg("Latent buildslave %s attached to %s" % (slave.slavename, 242 self.builder_name))

243

244 - def prepare(self, builder_status):

245 log.msg("substantiating slave %s" % (self,)) 246 d = self.substantiate() 247 def substantiation_failed(f): 248 builder_status.addPointEvent(['removing', 'latent', 249 self.slave.slavename]) 250 self.slave.disconnect() 251 # TODO: should failover to a new Build 252 return f

253 d.addErrback(substantiation_failed) 254 return d

255

256 - def substantiate(self):

257 self.state = SUBSTANTIATING 258 d = self.slave.substantiate(self) 259 if not self.slave.substantiated: 260 event = self.builder.builder_status.addEvent( 261 ["substantiating"]) 262 def substantiated(res): 263 msg = ["substantiate", "success"] 264 if isinstance(res, basestring): 265 msg.append(res) 266 elif isinstance(res, (tuple, list)): 267 msg.extend(res) 268 event.text = msg 269 event.finish() 270 return res

271 def substantiation_failed(res): 272 event.text = ["substantiate", "failed"] 273 # TODO add log of traceback to event 274 event.finish() 275 return res 276 d.addCallbacks(substantiated, substantiation_failed) 277 return d 278

279 - def detached(self):

280 AbstractSlaveBuilder.detached(self) 281 self.state = LATENT

282

283 - def buildStarted(self):

284 AbstractSlaveBuilder.buildStarted(self) 285 self.slave.buildStarted(self)

286

287 - def buildFinished(self):

288 AbstractSlaveBuilder.buildFinished(self) 289 self.slave.buildFinished(self)

290

291 - def _attachFailure(self, why, where):

292 self.state = LATENT 293 return AbstractSlaveBuilder._attachFailure(self, why, where)

294

295 - def ping(self, status=None):

296 if not self.slave.substantiated: 297 if status: 298 status.addEvent(["ping", "latent"]).finish() 299 return defer.succeed(True) 300 return AbstractSlaveBuilder.ping(self, status)

301 302

303 -class Builder(pb.Referenceable, service.MultiService):

304 """I manage all Builds of a given type. 305 306 Each Builder is created by an entry in the config file (the c['builders'] 307 list), with a number of parameters. 308 309 One of these parameters is the L{buildbot.process.factory.BuildFactory} 310 object that is associated with this Builder. The factory is responsible 311 for creating new L{Build<buildbot.process.base.Build>} objects. Each 312 Build object defines when and how the build is performed, so a new 313 Factory or Builder should be defined to control this behavior. 314 315 The Builder holds on to a number of L{base.BuildRequest} objects in a 316 list named C{.buildable}. Incoming BuildRequest objects will be added to 317 this list, or (if possible) merged into an existing request. When a slave 318 becomes available, I will use my C{BuildFactory} to turn the request into 319 a new C{Build} object. The C{BuildRequest} is forgotten, the C{Build} 320 goes into C{.building} while it runs. Once the build finishes, I will 321 discard it. 322 323 I maintain a list of available SlaveBuilders, one for each connected 324 slave that the C{slavenames} parameter says we can use. Some of these 325 will be idle, some of them will be busy running builds for me. If there 326 are multiple slaves, I can run multiple builds at once. 327 328 I also manage forced builds, progress expectation (ETA) management, and 329 some status delivery chores. 330 331 @type buildable: list of L{buildbot.process.base.BuildRequest} 332 @ivar buildable: BuildRequests that are ready to build, but which are 333 waiting for a buildslave to be available. 334 335 @type building: list of L{buildbot.process.base.Build} 336 @ivar building: Builds that are actively running 337 338 @type slaves: list of L{buildbot.buildslave.BuildSlave} objects 339 @ivar slaves: the slaves currently available for building 340 """ 341 342 expectations = None # this is created the first time we get a good build 343 CHOOSE_SLAVES_RANDOMLY = True # disabled for determinism during tests 344

345 - def __init__(self, setup, builder_status):

346 """ 347 @type setup: dict 348 @param setup: builder setup data, as stored in 349 BuildmasterConfig['builders']. Contains name, 350 slavename(s), builddir, slavebuilddir, factory, locks. 351 @type builder_status: L{buildbot.status.builder.BuilderStatus} 352 """ 353 service.MultiService.__init__(self) 354 self.name = setup['name'] 355 self.slavenames = [] 356 if setup.has_key('slavename'): 357 self.slavenames.append(setup['slavename']) 358 if setup.has_key('slavenames'): 359 self.slavenames.extend(setup['slavenames']) 360 self.builddir = setup['builddir'] 361 self.slavebuilddir = setup['slavebuilddir'] 362 self.buildFactory = setup['factory'] 363 self.nextSlave = setup.get('nextSlave') 364 if self.nextSlave is not None and not callable(self.nextSlave): 365 raise ValueError("nextSlave must be callable") 366 self.locks = setup.get("locks", []) 367 self.env = setup.get('env', {}) 368 assert isinstance(self.env, dict) 369 if setup.has_key('periodicBuildTime'): 370 raise ValueError("periodicBuildTime can no longer be defined as" 371 " part of the Builder: use scheduler.Periodic" 372 " instead") 373 self.nextBuild = setup.get('nextBuild') 374 if self.nextBuild is not None and not callable(self.nextBuild): 375 raise ValueError("nextBuild must be callable") 376 self.buildHorizon = setup.get('buildHorizon') 377 self.logHorizon = setup.get('logHorizon') 378 self.eventHorizon = setup.get('eventHorizon') 379 self.mergeRequests = setup.get('mergeRequests', True) 380 self.properties = setup.get('properties', {}) 381 382 # build/wannabuild slots: Build objects move along this sequence 383 self.building = [] 384 # old_building holds active builds that were stolen from a predecessor 385 self.old_building = weakref.WeakKeyDictionary() 386 387 # buildslaves which have connected but which are not yet available. 388 # These are always in the ATTACHING state. 389 self.attaching_slaves = [] 390 391 # buildslaves at our disposal. Each SlaveBuilder instance has a 392 # .state that is IDLE, PINGING, or BUILDING. "PINGING" is used when a 393 # Build is about to start, to make sure that they're still alive. 394 self.slaves = [] 395 396 self.builder_status = builder_status 397 self.builder_status.setSlavenames(self.slavenames) 398 self.builder_status.buildHorizon = self.buildHorizon 399 self.builder_status.logHorizon = self.logHorizon 400 self.builder_status.eventHorizon = self.eventHorizon 401 t = internet.TimerService(10*60, self.reclaimAllBuilds) 402 t.setServiceParent(self) 403 404 # for testing, to help synchronize tests 405 self.watchers = {'attach': [], 'detach': [], 'detach_all': [], 406 'idle': []} 407 self.run_count = 0

408

409 - def setBotmaster(self, botmaster):

410 self.botmaster = botmaster 411 self.db = botmaster.db 412 self.master_name = botmaster.master_name 413 self.master_incarnation = botmaster.master_incarnation

414

415 - def compareToSetup(self, setup):

416 diffs = [] 417 setup_slavenames = [] 418 if setup.has_key('slavename'): 419 setup_slavenames.append(setup['slavename']) 420 setup_slavenames.extend(setup.get('slavenames', [])) 421 if setup_slavenames != self.slavenames: 422 diffs.append('slavenames changed from %s to %s' \ 423 % (self.slavenames, setup_slavenames)) 424 if setup['builddir'] != self.builddir: 425 diffs.append('builddir changed from %s to %s' \ 426 % (self.builddir, setup['builddir'])) 427 if setup['slavebuilddir'] != self.slavebuilddir: 428 diffs.append('slavebuilddir changed from %s to %s' \ 429 % (self.slavebuilddir, setup['slavebuilddir'])) 430 if setup['factory'] != self.buildFactory: # compare objects 431 diffs.append('factory changed') 432 if setup.get('locks', []) != self.locks: 433 diffs.append('locks changed from %s to %s' % (self.locks, setup.get('locks'))) 434 if setup.get('nextSlave') != self.nextSlave: 435 diffs.append('nextSlave changed from %s to %s' % (self.nextSlave, setup.get('nextSlave'))) 436 if setup.get('nextBuild') != self.nextBuild: 437 diffs.append('nextBuild changed from %s to %s' % (self.nextBuild, setup.get('nextBuild'))) 438 if setup['buildHorizon'] != self.buildHorizon: 439 diffs.append('buildHorizon changed from %s to %s' % (self.buildHorizon, setup['buildHorizon'])) 440 if setup['logHorizon'] != self.logHorizon: 441 diffs.append('logHorizon changed from %s to %s' % (self.logHorizon, setup['logHorizon'])) 442 if setup['eventHorizon'] != self.eventHorizon: 443 diffs.append('eventHorizon changed from %s to %s' % (self.eventHorizon, setup['eventHorizon'])) 444 return diffs

445

446 - def __repr__(self):

447 return "<Builder '%r' at %d>" % (self.name, id(self))

448

449 - def triggerNewBuildCheck(self):

450 self.botmaster.triggerNewBuildCheck()

451

452 - def run(self):

453 """Check for work to be done. This should be called any time I might 454 be able to start a job: 455 456 - when the Builder is first created 457 - when a new job has been added to the [buildrequests] DB table 458 - when a slave has connected 459 460 If I have both an available slave and the database contains a 461 BuildRequest that I can handle, I will claim the BuildRequest and 462 start the build. When the build finishes, I will retire the 463 BuildRequest. 464 """ 465 # overall plan: 466 # move .expectations to DB 467 468 assert self.running 469 log.msg("Builder.run %s: %s" % (self, self.slaves)) 470 self.run_count += 1 471 472 available_slaves = [sb for sb in self.slaves if sb.isAvailable()] 473 if not available_slaves: 474 self.updateBigStatus() 475 return 476 d = self.db.runInteraction(self._claim_buildreqs, available_slaves) 477 d.addCallback(self._start_builds) 478 return d

479 480 # slave-managers must refresh their claim on a build at least once an 481 # hour, less any inter-manager clock skew 482 RECLAIM_INTERVAL = 1*3600 483

484 - def _claim_buildreqs(self, t, available_slaves):

485 # return a dict mapping slave -> (brid,ssid) 486 now = util.now() 487 old = now - self.RECLAIM_INTERVAL 488 requests = self.db.get_unclaimed_buildrequests(self.name, old, 489 self.master_name, 490 self.master_incarnation, 491 t) 492 493 assignments = {} 494 while requests and available_slaves: 495 sb = self._choose_slave(available_slaves) 496 if not sb: 497 log.msg("%s: want to start build, but we don't have a remote" 498 % self) 499 break 500 available_slaves.remove(sb) 501 breq = self._choose_build(requests) 502 if not breq: 503 log.msg("%s: went to start build, but nextBuild said not to" 504 % self) 505 break 506 requests.remove(breq) 507 merged_requests = [breq] 508 for other_breq in requests[:]: 509 if (self.mergeRequests and 510 self.botmaster.shouldMergeRequests(self, breq, other_breq) 511 ): 512 requests.remove(other_breq) 513 merged_requests.append(other_breq) 514 assignments[sb] = merged_requests 515 brids = [br.id for br in merged_requests] 516 self.db.claim_buildrequests(now, self.master_name, 517 self.master_incarnation, brids, t) 518 return assignments

519

520 - def _choose_slave(self, available_slaves):

521 # note: this might return None if the nextSlave() function decided to 522 # not give us anything 523 if self.nextSlave: 524 try: 525 return self.nextSlave(self, available_slaves) 526 except: 527 log.msg("Exception choosing next slave") 528 log.err(Failure()) 529 return None 530 if self.CHOOSE_SLAVES_RANDOMLY: 531 return random.choice(available_slaves) 532 return available_slaves[0]

533

534 - def _choose_build(self, buildable):

535 if self.nextBuild: 536 try: 537 return self.nextBuild(self, buildable) 538 except: 539 log.msg("Exception choosing next build") 540 log.err(Failure()) 541 return None 542 return buildable[0]

543

544 - def _start_builds(self, assignments):

545 # because _claim_buildreqs runs in a separate thread, we might have 546 # lost a slave by this point. We treat that case the same as if we 547 # lose the slave right after the build starts: the initial ping 548 # fails. 549 for (sb, requests) in assignments.items(): 550 build = self.buildFactory.newBuild(requests) 551 build.setBuilder(self) 552 build.setLocks(self.locks) 553 if len(self.env) > 0: 554 build.setSlaveEnvironment(self.env) 555 self.startBuild(build, sb) 556 self.updateBigStatus()

557 558

559 - def getBuildable(self, limit=None):

560 return self.db.runInteractionNow(self._getBuildable, limit)

561 - def _getBuildable(self, t, limit):

562 now = util.now() 563 old = now - self.RECLAIM_INTERVAL 564 return self.db.get_unclaimed_buildrequests(self.name, old, 565 self.master_name, 566 self.master_incarnation, 567 t, 568 limit)

569

570 - def getOldestRequestTime(self):

571 """Returns the timestamp of the oldest build request for this builder. 572 573 If there are no build requests, None is returned.""" 574 buildable = self.getBuildable(1) 575 if buildable: 576 # TODO: this is sorted by priority first, not strictly reqtime 577 return buildable[0].getSubmitTime() 578 return None

579

580 - def cancelBuildRequest(self, brid):

581 return self.db.cancel_buildrequests([brid])

582

583 - def consumeTheSoulOfYourPredecessor(self, old):

584 """Suck the brain out of an old Builder. 585 586 This takes all the runtime state from an existing Builder and moves 587 it into ourselves. This is used when a Builder is changed in the 588 master.cfg file: the new Builder has a different factory, but we want 589 all the builds that were queued for the old one to get processed by 590 the new one. Any builds which are already running will keep running. 591 The new Builder will get as many of the old SlaveBuilder objects as 592 it wants.""" 593 594 log.msg("consumeTheSoulOfYourPredecessor: %s feeding upon %s" % 595 (self, old)) 596 # all pending builds are stored in the DB, so we don't have to do 597 # anything to claim them. The old builder will be stopService'd, 598 # which should make sure they don't start any new work 599 600 # old.building (i.e. builds which are still running) is not migrated 601 # directly: it keeps track of builds which were in progress in the 602 # old Builder. When those builds finish, the old Builder will be 603 # notified, not us. However, since the old SlaveBuilder will point to 604 # us, it is our maybeStartBuild() that will be triggered. 605 if old.building: 606 self.builder_status.setBigState("building") 607 # however, we do grab a weakref to the active builds, so that our 608 # BuilderControl can see them and stop them. We use a weakref because 609 # we aren't the one to get notified, so there isn't a convenient 610 # place to remove it from self.building . 611 for b in old.building: 612 self.old_building[b] = None 613 for b in old.old_building: 614 self.old_building[b] = None 615 616 # Our set of slavenames may be different. Steal any of the old 617 # buildslaves that we want to keep using. 618 for sb in old.slaves[:]: 619 if sb.slave.slavename in self.slavenames: 620 log.msg(" stealing buildslave %s" % sb) 621 self.slaves.append(sb) 622 old.slaves.remove(sb) 623 sb.setBuilder(self) 624 625 # old.attaching_slaves: 626 # these SlaveBuilders are waiting on a sequence of calls: 627 # remote.setMaster and remote.print . When these two complete, 628 # old._attached will be fired, which will add a 'connect' event to 629 # the builder_status and try to start a build. However, we've pulled 630 # everything out of the old builder's queue, so it will have no work 631 # to do. The outstanding remote.setMaster/print call will be holding 632 # the last reference to the old builder, so it will disappear just 633 # after that response comes back. 634 # 635 # The BotMaster will ask the slave to re-set their list of Builders 636 # shortly after this function returns, which will cause our 637 # attached() method to be fired with a bunch of references to remote 638 # SlaveBuilders, some of which we already have (by stealing them 639 # from the old Builder), some of which will be new. The new ones 640 # will be re-attached. 641 642 # Therefore, we don't need to do anything about old.attaching_slaves 643 644 return # all done

645

646 - def reclaimAllBuilds(self):

647 now = util.now() 648 brids = set() 649 for b in self.building: 650 brids.update([br.id for br in b.requests]) 651 for b in self.old_building: 652 brids.update([br.id for br in b.requests]) 653 self.db.claim_buildrequests(now, self.master_name, 654 self.master_incarnation, brids)

655

656 - def getBuild(self, number):

657 for b in self.building: 658 if b.build_status and b.build_status.number == number: 659 return b 660 for b in self.old_building.keys(): 661 if b.build_status and b.build_status.number == number: 662 return b 663 return None

664

665 - def fireTestEvent(self, name, fire_with=None):

666 if fire_with is None: 667 fire_with = self 668 watchers = self.watchers[name] 669 self.watchers[name] = [] 670 for w in watchers: 671 eventually(w.callback, fire_with)

672

673 - def addLatentSlave(self, slave):

674 assert interfaces.ILatentBuildSlave.providedBy(slave) 675 for s in self.slaves: 676 if s == slave: 677 break 678 else: 679 sb = LatentSlaveBuilder(slave, self) 680 self.builder_status.addPointEvent( 681 ['added', 'latent', slave.slavename]) 682 self.slaves.append(sb) 683 self.triggerNewBuildCheck()

684

685 - def attached(self, slave, remote, commands):

686 """This is invoked by the BuildSlave when the self.slavename bot 687 registers their builder. 688 689 @type slave: L{buildbot.buildslave.BuildSlave} 690 @param slave: the BuildSlave that represents the buildslave as a whole 691 @type remote: L{twisted.spread.pb.RemoteReference} 692 @param remote: a reference to the L{buildbot.slave.bot.SlaveBuilder} 693 @type commands: dict: string -> string, or None 694 @param commands: provides the slave's version of each RemoteCommand 695 696 @rtype: L{twisted.internet.defer.Deferred} 697 @return: a Deferred that fires (with 'self') when the slave-side 698 builder is fully attached and ready to accept commands. 699 """ 700 for s in self.attaching_slaves + self.slaves: 701 if s.slave == slave: 702 # already attached to them. This is fairly common, since 703 # attached() gets called each time we receive the builder 704 # list from the slave, and we ask for it each time we add or 705 # remove a builder. So if the slave is hosting builders 706 # A,B,C, and the config file changes A, we'll remove A and 707 # re-add it, triggering two builder-list requests, getting 708 # two redundant calls to attached() for B, and another two 709 # for C. 710 # 711 # Therefore, when we see that we're already attached, we can 712 # just ignore it. TODO: build a diagram of the state 713 # transitions here, I'm concerned about sb.attached() failing 714 # and leaving sb.state stuck at 'ATTACHING', and about 715 # the detached() message arriving while there's some 716 # transition pending such that the response to the transition 717 # re-vivifies sb 718 return defer.succeed(self) 719 720 sb = SlaveBuilder() 721 sb.setBuilder(self) 722 self.attaching_slaves.append(sb) 723 d = sb.attached(slave, remote, commands) 724 d.addCallback(self._attached) 725 d.addErrback(self._not_attached, slave) 726 return d

727

728 - def _attached(self, sb):

729 # TODO: make this .addSlaveEvent(slave.slavename, ['connect']) ? 730 self.builder_status.addPointEvent(['connect', sb.slave.slavename]) 731 self.attaching_slaves.remove(sb) 732 self.slaves.append(sb) 733 734 self.fireTestEvent('attach') 735 return self

736

737 - def _not_attached(self, why, slave):

738 # already log.err'ed by SlaveBuilder._attachFailure 739 # TODO: make this .addSlaveEvent? 740 # TODO: remove from self.slaves (except that detached() should get 741 # run first, right?) 742 self.builder_status.addPointEvent(['failed', 'connect', 743 slave.slave.slavename]) 744 # TODO: add an HTMLLogFile of the exception 745 self.fireTestEvent('attach', why)

746

747 - def detached(self, slave):

748 """This is called when the connection to the bot is lost.""" 749 for sb in self.attaching_slaves + self.slaves: 750 if sb.slave == slave: 751 break 752 else: 753 log.msg("WEIRD: Builder.detached(%s) (%s)" 754 " not in attaching_slaves(%s)" 755 " or slaves(%s)" % (slave, slave.slavename, 756 self.attaching_slaves, 757 self.slaves)) 758 return 759 if sb.state == BUILDING: 760 # the Build's .lostRemote method (invoked by a notifyOnDisconnect 761 # handler) will cause the Build to be stopped, probably right 762 # after the notifyOnDisconnect that invoked us finishes running. 763 764 # TODO: should failover to a new Build 765 #self.retryBuild(sb.build) 766 pass 767 768 if sb in self.attaching_slaves: 769 self.attaching_slaves.remove(sb) 770 if sb in self.slaves: 771 self.slaves.remove(sb) 772 773 # TODO: make this .addSlaveEvent? 774 self.builder_status.addPointEvent(['disconnect', slave.slavename]) 775 sb.detached() # inform the SlaveBuilder that their slave went away 776 self.updateBigStatus() 777 self.fireTestEvent('detach') 778 if not self.slaves: 779 self.fireTestEvent('detach_all')

780

781 - def updateBigStatus(self):

782 if not self.slaves: 783 self.builder_status.setBigState("offline") 784 elif self.building: 785 self.builder_status.setBigState("building") 786 else: 787 self.builder_status.setBigState("idle") 788 self.fireTestEvent('idle')

789

790 - def startBuild(self, build, sb):

791 """Start a build on the given slave. 792 @param build: the L{base.Build} to start 793 @param sb: the L{SlaveBuilder} which will host this build 794 795 @return: a Deferred which fires with a 796 L{buildbot.interfaces.IBuildControl} that can be used to stop the 797 Build, or to access a L{buildbot.interfaces.IBuildStatus} which will 798 watch the Build as it runs. """ 799 800 self.building.append(build) 801 self.updateBigStatus() 802 log.msg("starting build %s using slave %s" % (build, sb)) 803 d = sb.prepare(self.builder_status) 804 def _ping(ign): 805 # ping the slave to make sure they're still there. If they've 806 # fallen off the map (due to a NAT timeout or something), this 807 # will fail in a couple of minutes, depending upon the TCP 808 # timeout. 809 # 810 # TODO: This can unnecessarily suspend the starting of a build, in 811 # situations where the slave is live but is pushing lots of data to 812 # us in a build. 813 log.msg("starting build %s.. pinging the slave %s" % (build, sb)) 814 return sb.ping()

815 d.addCallback(_ping) 816 d.addCallback(self._startBuild_1, build, sb) 817 return d

818

819 - def _startBuild_1(self, res, build, sb):

820 if not res: 821 return self._startBuildFailed("slave ping failed", build, sb) 822 # The buildslave is ready to go. sb.buildStarted() sets its state to 823 # BUILDING (so we won't try to use it for any other builds). This 824 # gets set back to IDLE by the Build itself when it finishes. 825 sb.buildStarted() 826 d = sb.remote.callRemote("startBuild") 827 d.addCallbacks(self._startBuild_2, self._startBuildFailed, 828 callbackArgs=(build,sb), errbackArgs=(build,sb)) 829 return d

830

831 - def _startBuild_2(self, res, build, sb):

832 # create the BuildStatus object that goes with the Build 833 bs = self.builder_status.newBuild() 834 835 # start the build. This will first set up the steps, then tell the 836 # BuildStatus that it has started, which will announce it to the 837 # world (through our BuilderStatus object, which is its parent). 838 # Finally it will start the actual build process. 839 bids = [self.db.build_started(req.id, bs.number) for req in build.requests] 840 d = build.startBuild(bs, self.expectations, sb) 841 d.addCallback(self.buildFinished, sb, bids) 842 # this shouldn't happen. if it does, the slave will be wedged 843 d.addErrback(log.err) 844 return build # this is the IBuildControl

845

846 - def _startBuildFailed(self, why, build, sb):

847 # put the build back on the buildable list 848 log.msg("I tried to tell the slave that the build %s started, but " 849 "remote_startBuild failed: %s" % (build, why)) 850 # release the slave. This will queue a call to maybeStartBuild, which 851 # will fire after other notifyOnDisconnect handlers have marked the 852 # slave as disconnected (so we don't try to use it again). 853 sb.buildFinished() 854 855 log.msg("re-queueing the BuildRequest") 856 self.building.remove(build) 857 self._resubmit_buildreqs(build).addErrback(log.err)

858

859 - def setupProperties(self, props):

860 props.setProperty("buildername", self.name, "Builder") 861 if len(self.properties) > 0: 862 for propertyname in self.properties: 863 props.setProperty(propertyname, self.properties[propertyname], "Builder")

864

865 - def buildFinished(self, build, sb, bids):

866 """This is called when the Build has finished (either success or 867 failure). Any exceptions during the build are reported with 868 results=FAILURE, not with an errback.""" 869 870 # by the time we get here, the Build has already released the slave 871 # (which queues a call to maybeStartBuild) 872 873 self.db.builds_finished(bids) 874 875 results = build.build_status.getResults() 876 self.building.remove(build) 877 if results == RETRY: 878 self._resubmit_buildreqs(build).addErrback(log.err) # returns Deferred 879 else: 880 brids = [br.id for br in build.requests] 881 self.db.retire_buildrequests(brids, results) 882 self.triggerNewBuildCheck()

883

884 - def _resubmit_buildreqs(self, build):

885 brids = [br.id for br in build.requests] 886 return self.db.resubmit_buildrequests(brids)

887

888 - def setExpectations(self, progress):

889 """Mark the build as successful and update expectations for the next 890 build. Only call this when the build did not fail in any way that 891 would invalidate the time expectations generated by it. (if the 892 compile failed and thus terminated early, we can't use the last 893 build to predict how long the next one will take). 894 """ 895 if self.expectations: 896 self.expectations.update(progress) 897 else: 898 # the first time we get a good build, create our Expectations 899 # based upon its results 900 self.expectations = Expectations(progress) 901 log.msg("new expectations: %s seconds" % \ 902 self.expectations.expectedBuildTime())

903

904 - def shutdownSlave(self):

905 if self.remote: 906 self.remote.callRemote("shutdown")

907 908

909 -class BuilderControl:

910 implements(interfaces.IBuilderControl) 911

912 - def __init__(self, builder, parent):

913 self.original = builder 914 self.parent = parent # the IControl object

915

916 - def submitBuildRequest(self, ss, reason, props=None, now=False):

917 bss = self.parent.submitBuildSet([self.original.name], ss, reason, 918 props, now) 919 brs = bss.getBuildRequests()[0] 920 return brs

921

922 - def rebuildBuild(self, bs, reason="<rebuild, no reason given>", extraProperties=None):

923 if not bs.isFinished(): 924 return 925 926 ss = bs.getSourceStamp(absolute=True) 927 if extraProperties is None: 928 properties = bs.getProperties() 929 else: 930 # Make a copy so as not to modify the original build. 931 properties = Properties() 932 properties.updateFromProperties(bs.getProperties()) 933 properties.updateFromProperties(extraProperties) 934 self.submitBuildRequest(ss, reason, props=properties)

935

936 - def getPendingBuilds(self):

937 # return IBuildRequestControl objects 938 retval = [] 939 for r in self.original.getBuildable(): 940 retval.append(BuildRequestControl(self.original, r)) 941 942 return retval

943

944 - def getBuild(self, number):

945 return self.original.getBuild(number)

946

947 - def ping(self):

948 if not self.original.slaves: 949 self.original.builder_status.addPointEvent(["ping", "no slave"]) 950 return defer.succeed(False) # interfaces.NoSlaveError 951 dl = [] 952 for s in self.original.slaves: 953 dl.append(s.ping(self.original.builder_status)) 954 d = defer.DeferredList(dl) 955 d.addCallback(self._gatherPingResults) 956 return d

957

958 - def _gatherPingResults(self, res):

959 for ignored,success in res: 960 if not success: 961 return False 962 return True

963

964 -class BuildRequestControl:

965 implements(interfaces.IBuildRequestControl) 966

967 - def __init__(self, builder, request):

968 self.original_builder = builder 969 self.original_request = request 970 self.brid = request.id

971 974

975 - def unsubscribe(self, observer):

976 raise NotImplementedError

977

978 - def cancel(self):

979 self.original_builder.cancelBuildRequest(self.brid)

980

Source Code for Module buildbot.process.builder