buildbot.process.builder

1 2 import random, weakref 3 from zope.interface import implements 4 from twisted.python import log 5 from twisted.python.failure import Failure 6 from twisted.spread import pb 7 from twisted.application import service, internet 8 from twisted.internet import defer 9 10 from buildbot import interfaces, util 11 from buildbot.status.progress import Expectations 12 from buildbot.status.builder import RETRY 13 from buildbot.process.properties import Properties 14 from buildbot.util.eventual import eventually 15 16 (ATTACHING, # slave attached, still checking hostinfo/etc 17 IDLE, # idle, available for use 18 PINGING, # build about to start, making sure it is still alive 19 BUILDING, # build is running 20 LATENT, # latent slave is not substantiated; similar to idle 21 SUBSTANTIATING, 22 ) = range(6) 23 24

25 -class AbstractSlaveBuilder(pb.Referenceable):

26 """I am the master-side representative for one of the 27 L{buildbot.slave.bot.SlaveBuilder} objects that lives in a remote 28 buildbot. When a remote builder connects, I query it for command versions 29 and then make it available to any Builds that are ready to run. """ 30

31 - def __init__(self):

32 self.ping_watchers = [] 33 self.state = None # set in subclass 34 self.remote = None 35 self.slave = None 36 self.builder_name = None

37

38 - def __repr__(self):

39 r = ["<", self.__class__.__name__] 40 if self.builder_name: 41 r.extend([" builder=", repr(self.builder_name)]) 42 if self.slave: 43 r.extend([" slave=", repr(self.slave.slavename)]) 44 r.append(">") 45 return ''.join(r)

46

47 - def setBuilder(self, b):

48 self.builder = b 49 self.builder_name = b.name

50

51 - def getSlaveCommandVersion(self, command, oldversion=None):

52 if self.remoteCommands is None: 53 # the slave is 0.5.0 or earlier 54 return oldversion 55 return self.remoteCommands.get(command)

56

57 - def isAvailable(self):

58 # if this SlaveBuilder is busy, then it's definitely not available 59 if self.isBusy(): 60 return False 61 62 # otherwise, check in with the BuildSlave 63 if self.slave: 64 return self.slave.canStartBuild() 65 66 # no slave? not very available. 67 return False

68

69 - def isBusy(self):

70 return self.state not in (IDLE, LATENT)

71

72 - def buildStarted(self):

73 self.state = BUILDING

74

75 - def buildFinished(self):

76 self.state = IDLE 77 self.builder.triggerNewBuildCheck()

78

79 - def attached(self, slave, remote, commands):

80 """ 81 @type slave: L{buildbot.buildslave.BuildSlave} 82 @param slave: the BuildSlave that represents the buildslave as a 83 whole 84 @type remote: L{twisted.spread.pb.RemoteReference} 85 @param remote: a reference to the L{buildbot.slave.bot.SlaveBuilder} 86 @type commands: dict: string -> string, or None 87 @param commands: provides the slave's version of each RemoteCommand 88 """ 89 self.state = ATTACHING 90 self.remote = remote 91 self.remoteCommands = commands # maps command name to version 92 if self.slave is None: 93 self.slave = slave 94 self.slave.addSlaveBuilder(self) 95 else: 96 assert self.slave == slave 97 log.msg("Buildslave %s attached to %s" % (slave.slavename, 98 self.builder_name)) 99 d = self.remote.callRemote("setMaster", self) 100 d.addErrback(self._attachFailure, "Builder.setMaster") 101 d.addCallback(self._attached2) 102 return d

103

104 - def _attached2(self, res):

105 d = self.remote.callRemote("print", "attached") 106 d.addErrback(self._attachFailure, "Builder.print 'attached'") 107 d.addCallback(self._attached3) 108 return d

109

110 - def _attached3(self, res):

111 # now we say they're really attached 112 self.state = IDLE 113 return self

114

115 - def _attachFailure(self, why, where):

116 assert isinstance(where, str) 117 log.msg(where) 118 log.err(why) 119 return why

120

121 - def prepare(self, builder_status):

122 return defer.succeed(None)

123

124 - def ping(self, status=None):

125 """Ping the slave to make sure it is still there. Returns a Deferred 126 that fires with True if it is. 127 128 @param status: if you point this at a BuilderStatus, a 'pinging' 129 event will be pushed. 130 """ 131 oldstate = self.state 132 self.state = PINGING 133 newping = not self.ping_watchers 134 d = defer.Deferred() 135 self.ping_watchers.append(d) 136 if newping: 137 if status: 138 event = status.addEvent(["pinging"]) 139 d2 = defer.Deferred() 140 d2.addCallback(self._pong_status, event) 141 self.ping_watchers.insert(0, d2) 142 # I think it will make the tests run smoother if the status 143 # is updated before the ping completes 144 Ping().ping(self.remote).addCallback(self._pong) 145 146 def reset_state(res): 147 if self.state == PINGING: 148 self.state = oldstate 149 return res

150 d.addCallback(reset_state) 151 return d

152

153 - def _pong(self, res):

154 watchers, self.ping_watchers = self.ping_watchers, [] 155 for d in watchers: 156 d.callback(res)

157

158 - def _pong_status(self, res, event):

159 if res: 160 event.text = ["ping", "success"] 161 else: 162 event.text = ["ping", "failed"] 163 event.finish()

164

165 - def detached(self):

166 log.msg("Buildslave %s detached from %s" % (self.slave.slavename, 167 self.builder_name)) 168 if self.slave: 169 self.slave.removeSlaveBuilder(self) 170 self.slave = None 171 self.remote = None 172 self.remoteCommands = None

173 174

175 -class Ping:

176 running = False 177

178 - def ping(self, remote):

179 assert not self.running 180 if not remote: 181 # clearly the ping must fail 182 return defer.succeed(False) 183 self.running = True 184 log.msg("sending ping") 185 self.d = defer.Deferred() 186 # TODO: add a distinct 'ping' command on the slave.. using 'print' 187 # for this purpose is kind of silly. 188 remote.callRemote("print", "ping").addCallbacks(self._pong, 189 self._ping_failed, 190 errbackArgs=(remote,)) 191 return self.d

192

193 - def _pong(self, res):

194 log.msg("ping finished: success") 195 self.d.callback(True)

196

197 - def _ping_failed(self, res, remote):

198 log.msg("ping finished: failure") 199 # the slave has some sort of internal error, disconnect them. If we 200 # don't, we'll requeue a build and ping them again right away, 201 # creating a nasty loop. 202 remote.broker.transport.loseConnection() 203 # TODO: except, if they actually did manage to get this far, they'll 204 # probably reconnect right away, and we'll do this game again. Maybe 205 # it would be better to leave them in the PINGING state. 206 self.d.callback(False)

207 208

209 -class SlaveBuilder(AbstractSlaveBuilder):

210

211 - def __init__(self):

212 AbstractSlaveBuilder.__init__(self) 213 self.state = ATTACHING

214

215 - def detached(self):

216 AbstractSlaveBuilder.detached(self) 217 if self.slave: 218 self.slave.removeSlaveBuilder(self) 219 self.slave = None 220 self.state = ATTACHING

221

222 - def buildFinished(self):

223 # Call the slave's buildFinished if we can; the slave may be waiting 224 # to do a graceful shutdown and needs to know when it's idle. 225 # After, we check to see if we can start other builds. 226 self.state = IDLE 227 if self.slave: 228 d = self.slave.buildFinished(self) 229 d.addCallback(lambda x: self.builder.triggerNewBuildCheck()) 230 else: 231 self.builder.triggerNewBuildCheck()

232 233

234 -class LatentSlaveBuilder(AbstractSlaveBuilder):

235 - def __init__(self, slave, builder):

236 AbstractSlaveBuilder.__init__(self) 237 self.slave = slave 238 self.state = LATENT 239 self.setBuilder(builder) 240 self.slave.addSlaveBuilder(self) 241 log.msg("Latent buildslave %s attached to %s" % (slave.slavename, 242 self.builder_name))

243

244 - def prepare(self, builder_status):

245 log.msg("substantiating slave %s" % (self,)) 246 d = self.substantiate() 247 def substantiation_failed(f): 248 builder_status.addPointEvent(['removing', 'latent', 249 self.slave.slavename]) 250 self.slave.disconnect() 251 # TODO: should failover to a new Build 252 return f

253 d.addErrback(substantiation_failed) 254 return d

255

256 - def substantiate(self):

257 self.state = SUBSTANTIATING 258 d = self.slave.substantiate(self) 259 if not self.slave.substantiated: 260 event = self.builder.builder_status.addEvent( 261 ["substantiating"]) 262 def substantiated(res): 263 msg = ["substantiate", "success"] 264 if isinstance(res, basestring): 265 msg.append(res) 266 elif isinstance(res, (tuple, list)): 267 msg.extend(res) 268 event.text = msg 269 event.finish() 270 return res

271 def substantiation_failed(res): 272 event.text = ["substantiate", "failed"] 273 # TODO add log of traceback to event 274 event.finish() 275 return res 276 d.addCallbacks(substantiated, substantiation_failed) 277 return d 278

279 - def detached(self):

280 AbstractSlaveBuilder.detached(self) 281 self.state = LATENT

282

283 - def buildStarted(self):

284 AbstractSlaveBuilder.buildStarted(self) 285 self.slave.buildStarted(self)

286

287 - def buildFinished(self):

288 AbstractSlaveBuilder.buildFinished(self) 289 self.slave.buildFinished(self)

290

291 - def _attachFailure(self, why, where):

292 self.state = LATENT 293 return AbstractSlaveBuilder._attachFailure(self, why, where)

294

295 - def ping(self, status=None):

296 if not self.slave.substantiated: 297 if status: 298 status.addEvent(["ping", "latent"]).finish() 299 return defer.succeed(True) 300 return AbstractSlaveBuilder.ping(self, status)

301 302

303 -class Builder(pb.Referenceable, service.MultiService):

304 """I manage all Builds of a given type. 305 306 Each Builder is created by an entry in the config file (the c['builders'] 307 list), with a number of parameters. 308 309 One of these parameters is the L{buildbot.process.factory.BuildFactory} 310 object that is associated with this Builder. The factory is responsible 311 for creating new L{Build<buildbot.process.base.Build>} objects. Each 312 Build object defines when and how the build is performed, so a new 313 Factory or Builder should be defined to control this behavior. 314 315 The Builder holds on to a number of L{base.BuildRequest} objects in a 316 list named C{.buildable}. Incoming BuildRequest objects will be added to 317 this list, or (if possible) merged into an existing request. When a slave 318 becomes available, I will use my C{BuildFactory} to turn the request into 319 a new C{Build} object. The C{BuildRequest} is forgotten, the C{Build} 320 goes into C{.building} while it runs. Once the build finishes, I will 321 discard it. 322 323 I maintain a list of available SlaveBuilders, one for each connected 324 slave that the C{slavenames} parameter says we can use. Some of these 325 will be idle, some of them will be busy running builds for me. If there 326 are multiple slaves, I can run multiple builds at once. 327 328 I also manage forced builds, progress expectation (ETA) management, and 329 some status delivery chores. 330 331 @type buildable: list of L{buildbot.process.base.BuildRequest} 332 @ivar buildable: BuildRequests that are ready to build, but which are 333 waiting for a buildslave to be available. 334 335 @type building: list of L{buildbot.process.base.Build} 336 @ivar building: Builds that are actively running 337 338 @type slaves: list of L{buildbot.buildslave.BuildSlave} objects 339 @ivar slaves: the slaves currently available for building 340 """ 341 342 expectations = None # this is created the first time we get a good build 343 CHOOSE_SLAVES_RANDOMLY = True # disabled for determinism during tests 344

345 - def __init__(self, setup, builder_status):

346 """ 347 @type setup: dict 348 @param setup: builder setup data, as stored in 349 BuildmasterConfig['builders']. Contains name, 350 slavename(s), builddir, slavebuilddir, factory, locks. 351 @type builder_status: L{buildbot.status.builder.BuilderStatus} 352 """ 353 service.MultiService.__init__(self) 354 self.name = setup['name'] 355 self.slavenames = [] 356 if setup.has_key('slavename'): 357 self.slavenames.append(setup['slavename']) 358 if setup.has_key('slavenames'): 359 self.slavenames.extend(setup['slavenames']) 360 self.builddir = setup['builddir'] 361 self.slavebuilddir = setup['slavebuilddir'] 362 self.buildFactory = setup['factory'] 363 self.nextSlave = setup.get('nextSlave') 364 if self.nextSlave is not None and not callable(self.nextSlave): 365 raise ValueError("nextSlave must be callable") 366 self.locks = setup.get("locks", []) 367 self.env = setup.get('env', {}) 368 assert isinstance(self.env, dict) 369 if setup.has_key('periodicBuildTime'): 370 raise ValueError("periodicBuildTime can no longer be defined as" 371 " part of the Builder: use scheduler.Periodic" 372 " instead") 373 self.nextBuild = setup.get('nextBuild') 374 if self.nextBuild is not None and not callable(self.nextBuild): 375 raise ValueError("nextBuild must be callable") 376 self.buildHorizon = setup.get('buildHorizon') 377 self.logHorizon = setup.get('logHorizon') 378 self.eventHorizon = setup.get('eventHorizon') 379 self.mergeRequests = setup.get('mergeRequests', True) 380 self.properties = setup.get('properties', {}) 381 382 # build/wannabuild slots: Build objects move along this sequence 383 self.building = [] 384 # old_building holds active builds that were stolen from a predecessor 385 self.old_building = weakref.WeakKeyDictionary() 386 387 # buildslaves which have connected but which are not yet available. 388 # These are always in the ATTACHING state. 389 self.attaching_slaves = [] 390 391 # buildslaves at our disposal. Each SlaveBuilder instance has a 392 # .state that is IDLE, PINGING, or BUILDING. "PINGING" is used when a 393 # Build is about to start, to make sure that they're still alive. 394 self.slaves = [] 395 396 self.builder_status = builder_status 397 self.builder_status.setSlavenames(self.slavenames) 398 self.builder_status.buildHorizon = self.buildHorizon 399 self.builder_status.logHorizon = self.logHorizon 400 self.builder_status.eventHorizon = self.eventHorizon 401 t = internet.TimerService(10*60, self.reclaimAllBuilds) 402 t.setServiceParent(self) 403 404 # for testing, to help synchronize tests 405 self.watchers = {'attach': [], 'detach': [], 'detach_all': [], 406 'idle': []} 407 self.run_count = 0

408

409 - def setBotmaster(self, botmaster):

410 self.botmaster = botmaster 411 self.db = botmaster.db 412 self.master_name = botmaster.master_name 413 self.master_incarnation = botmaster.master_incarnation

414

415 - def compareToSetup(self, setup):

416 diffs = [] 417 setup_slavenames = [] 418 if setup.has_key('slavename'): 419 setup_slavenames.append(setup['slavename']) 420 setup_slavenames.extend(setup.get('slavenames', [])) 421 if setup_slavenames != self.slavenames: 422 diffs.append('slavenames changed from %s to %s' \ 423 % (self.slavenames, setup_slavenames)) 424 if setup['builddir'] != self.builddir: 425 diffs.append('builddir changed from %s to %s' \ 426 % (self.builddir, setup['builddir'])) 427 if setup['slavebuilddir'] != self.slavebuilddir: 428 diffs.append('slavebuilddir changed from %s to %s' \ 429 % (self.slavebuilddir, setup['slavebuilddir'])) 430 if setup['factory'] != self.buildFactory: # compare objects 431 diffs.append('factory changed') 432 if setup.get('locks', []) != self.locks: 433 diffs.append('locks changed from %s to %s' % (self.locks, setup.get('locks'))) 434 if setup.get('nextSlave') != self.nextSlave: 435 diffs.append('nextSlave changed from %s to %s' % (self.nextSlave, setup.get('nextSlave'))) 436 if setup.get('nextBuild') != self.nextBuild: 437 diffs.append('nextBuild changed from %s to %s' % (self.nextBuild, setup.get('nextBuild'))) 438 if setup['buildHorizon'] != self.buildHorizon: 439 diffs.append('buildHorizon changed from %s to %s' % (self.buildHorizon, setup['buildHorizon'])) 440 if setup['logHorizon'] != self.logHorizon: 441 diffs.append('logHorizon changed from %s to %s' % (self.logHorizon, setup['logHorizon'])) 442 if setup['eventHorizon'] != self.eventHorizon: 443 diffs.append('eventHorizon changed from %s to %s' % (self.eventHorizon, setup['eventHorizon'])) 444 return diffs

445

446 - def __repr__(self):

447 return "<Builder '%r' at %d>" % (self.name, id(self))

448

449 - def triggerNewBuildCheck(self):

450 self.botmaster.triggerNewBuildCheck()

451

452 - def run(self):

453 """Check for work to be done. This should be called any time I might 454 be able to start a job: 455 456 - when the Builder is first created 457 - when a new job has been added to the [buildrequests] DB table 458 - when a slave has connected 459 460 If I have both an available slave and the database contains a 461 BuildRequest that I can handle, I will claim the BuildRequest and 462 start the build. When the build finishes, I will retire the 463 BuildRequest. 464 """ 465 # overall plan: 466 # move .expectations to DB 467 468 assert self.running 469 log.msg("Builder.run %s: %s" % (self, self.slaves)) 470 self.run_count += 1 471 472 available_slaves = [sb for sb in self.slaves if sb.isAvailable()] 473 if not available_slaves: 474 self.updateBigStatus() 475 return 476 d = self.db.runInteraction(self._claim_buildreqs, available_slaves) 477 d.addCallback(self._start_builds) 478 return d

479 480 # slave-managers must refresh their claim on a build at least once an 481 # hour, less any inter-manager clock skew 482 RECLAIM_INTERVAL = 1*3600 483

484 - def _claim_buildreqs(self, t, available_slaves):

485 # return a dict mapping slave -> (brid,ssid) 486 now = util.now() 487 old = now - self.RECLAIM_INTERVAL 488 requests = self.db.get_unclaimed_buildrequests(self.name, old, 489 self.master_name, 490 self.master_incarnation, 491 t) 492 493 assignments = {} 494 while requests and available_slaves: 495 sb = self._choose_slave(available_slaves) 496 if not sb: 497 log.msg("%s: want to start build, but we don't have a remote" 498 % self) 499 break 500 available_slaves.remove(sb) 501 breq = self._choose_build(requests) 502 if not breq: 503 log.msg("%s: went to start build, but nextBuild said not to" 504 % self) 505 break 506 requests.remove(breq) 507 merged_requests = [breq] 508 for other_breq in requests[:]: 509 if (self.mergeRequests and 510 self.botmaster.shouldMergeRequests(self, breq, other_breq) 511 ): 512 requests.remove(other_breq) 513 merged_requests.append(other_breq) 514 assignments[sb] = merged_requests 515 brids = [br.id for br in merged_requests] 516 self.db.claim_buildrequests(now, self.master_name, 517 self.master_incarnation, brids, t) 518 return assignments

519

520 - def _choose_slave(self, available_slaves):

521 # note: this might return None if the nextSlave() function decided to 522 # not give us anything 523 if self.nextSlave: 524 try: 525 return self.nextSlave(self, available_slaves) 526 except: 527 log.msg("Exception choosing next slave") 528 log.err(Failure()) 529 return None 530 if self.CHOOSE_SLAVES_RANDOMLY: 531 return random.choice(available_slaves) 532 return available_slaves[0]

533

534 - def _choose_build(self, buildable):

535 if self.nextBuild: 536 try: 537 return self.nextBuild(self, buildable) 538 except: 539 log.msg("Exception choosing next build") 540 log.err(Failure()) 541 return None 542 return buildable[0]

543

544 - def _start_builds(self, assignments):

545 # because _claim_buildreqs runs in a separate thread, we might have 546 # lost a slave by this point. We treat that case the same as if we 547 # lose the slave right after the build starts: the initial ping 548 # fails. 549 for (sb, requests) in assignments.items(): 550 build = self.buildFactory.newBuild(requests) 551 build.setBuilder(self) 552 build.setLocks(self.locks) 553 if len(self.env) > 0: 554 build.setSlaveEnvironment(self.env) 555 self.startBuild(build, sb) 556 self.updateBigStatus()

557 558

559 - def getBuildable(self):

560 return self.db.runInteractionNow(self._getBuildable)

561 - def _getBuildable(self, t):

562 now = util.now() 563 old = now - self.RECLAIM_INTERVAL 564 return self.db.get_unclaimed_buildrequests(self.name, old, 565 self.master_name, 566 self.master_incarnation, 567 t)

568

569 - def getOldestRequestTime(self):

570 """Returns the timestamp of the oldest build request for this builder. 571 572 If there are no build requests, None is returned.""" 573 buildable = self.getBuildable() 574 if buildable: 575 # TODO: this is sorted by priority first, not strictly reqtime 576 return buildable[0].getSubmitTime() 577 return None

578

579 - def cancelBuildRequest(self, brid):

580 return self.db.cancel_buildrequests([brid])

581

582 - def consumeTheSoulOfYourPredecessor(self, old):

583 """Suck the brain out of an old Builder. 584 585 This takes all the runtime state from an existing Builder and moves 586 it into ourselves. This is used when a Builder is changed in the 587 master.cfg file: the new Builder has a different factory, but we want 588 all the builds that were queued for the old one to get processed by 589 the new one. Any builds which are already running will keep running. 590 The new Builder will get as many of the old SlaveBuilder objects as 591 it wants.""" 592 593 log.msg("consumeTheSoulOfYourPredecessor: %s feeding upon %s" % 594 (self, old)) 595 # all pending builds are stored in the DB, so we don't have to do 596 # anything to claim them. The old builder will be stopService'd, 597 # which should make sure they don't start any new work 598 599 # old.building (i.e. builds which are still running) is not migrated 600 # directly: it keeps track of builds which were in progress in the 601 # old Builder. When those builds finish, the old Builder will be 602 # notified, not us. However, since the old SlaveBuilder will point to 603 # us, it is our maybeStartBuild() that will be triggered. 604 if old.building: 605 self.builder_status.setBigState("building") 606 # however, we do grab a weakref to the active builds, so that our 607 # BuilderControl can see them and stop them. We use a weakref because 608 # we aren't the one to get notified, so there isn't a convenient 609 # place to remove it from self.building . 610 for b in old.building: 611 self.old_building[b] = None 612 for b in old.old_building: 613 self.old_building[b] = None 614 615 # Our set of slavenames may be different. Steal any of the old 616 # buildslaves that we want to keep using. 617 for sb in old.slaves[:]: 618 if sb.slave.slavename in self.slavenames: 619 log.msg(" stealing buildslave %s" % sb) 620 self.slaves.append(sb) 621 old.slaves.remove(sb) 622 sb.setBuilder(self) 623 624 # old.attaching_slaves: 625 # these SlaveBuilders are waiting on a sequence of calls: 626 # remote.setMaster and remote.print . When these two complete, 627 # old._attached will be fired, which will add a 'connect' event to 628 # the builder_status and try to start a build. However, we've pulled 629 # everything out of the old builder's queue, so it will have no work 630 # to do. The outstanding remote.setMaster/print call will be holding 631 # the last reference to the old builder, so it will disappear just 632 # after that response comes back. 633 # 634 # The BotMaster will ask the slave to re-set their list of Builders 635 # shortly after this function returns, which will cause our 636 # attached() method to be fired with a bunch of references to remote 637 # SlaveBuilders, some of which we already have (by stealing them 638 # from the old Builder), some of which will be new. The new ones 639 # will be re-attached. 640 641 # Therefore, we don't need to do anything about old.attaching_slaves 642 643 return # all done

644

645 - def reclaimAllBuilds(self):

646 now = util.now() 647 brids = set() 648 for b in self.building: 649 brids.update([br.id for br in b.requests]) 650 for b in self.old_building: 651 brids.update([br.id for br in b.requests]) 652 self.db.claim_buildrequests(now, self.master_name, 653 self.master_incarnation, brids)

654

655 - def getBuild(self, number):

656 for b in self.building: 657 if b.build_status and b.build_status.number == number: 658 return b 659 for b in self.old_building.keys(): 660 if b.build_status and b.build_status.number == number: 661 return b 662 return None

663

664 - def fireTestEvent(self, name, fire_with=None):

665 if fire_with is None: 666 fire_with = self 667 watchers = self.watchers[name] 668 self.watchers[name] = [] 669 for w in watchers: 670 eventually(w.callback, fire_with)

671

672 - def addLatentSlave(self, slave):

673 assert interfaces.ILatentBuildSlave.providedBy(slave) 674 for s in self.slaves: 675 if s == slave: 676 break 677 else: 678 sb = LatentSlaveBuilder(slave, self) 679 self.builder_status.addPointEvent( 680 ['added', 'latent', slave.slavename]) 681 self.slaves.append(sb) 682 self.triggerNewBuildCheck()

683

684 - def attached(self, slave, remote, commands):

685 """This is invoked by the BuildSlave when the self.slavename bot 686 registers their builder. 687 688 @type slave: L{buildbot.buildslave.BuildSlave} 689 @param slave: the BuildSlave that represents the buildslave as a whole 690 @type remote: L{twisted.spread.pb.RemoteReference} 691 @param remote: a reference to the L{buildbot.slave.bot.SlaveBuilder} 692 @type commands: dict: string -> string, or None 693 @param commands: provides the slave's version of each RemoteCommand 694 695 @rtype: L{twisted.internet.defer.Deferred} 696 @return: a Deferred that fires (with 'self') when the slave-side 697 builder is fully attached and ready to accept commands. 698 """ 699 for s in self.attaching_slaves + self.slaves: 700 if s.slave == slave: 701 # already attached to them. This is fairly common, since 702 # attached() gets called each time we receive the builder 703 # list from the slave, and we ask for it each time we add or 704 # remove a builder. So if the slave is hosting builders 705 # A,B,C, and the config file changes A, we'll remove A and 706 # re-add it, triggering two builder-list requests, getting 707 # two redundant calls to attached() for B, and another two 708 # for C. 709 # 710 # Therefore, when we see that we're already attached, we can 711 # just ignore it. TODO: build a diagram of the state 712 # transitions here, I'm concerned about sb.attached() failing 713 # and leaving sb.state stuck at 'ATTACHING', and about 714 # the detached() message arriving while there's some 715 # transition pending such that the response to the transition 716 # re-vivifies sb 717 return defer.succeed(self) 718 719 sb = SlaveBuilder() 720 sb.setBuilder(self) 721 self.attaching_slaves.append(sb) 722 d = sb.attached(slave, remote, commands) 723 d.addCallback(self._attached) 724 d.addErrback(self._not_attached, slave) 725 return d

726

727 - def _attached(self, sb):

728 # TODO: make this .addSlaveEvent(slave.slavename, ['connect']) ? 729 self.builder_status.addPointEvent(['connect', sb.slave.slavename]) 730 self.attaching_slaves.remove(sb) 731 self.slaves.append(sb) 732 733 self.fireTestEvent('attach') 734 return self

735

736 - def _not_attached(self, why, slave):

737 # already log.err'ed by SlaveBuilder._attachFailure 738 # TODO: make this .addSlaveEvent? 739 # TODO: remove from self.slaves (except that detached() should get 740 # run first, right?) 741 self.builder_status.addPointEvent(['failed', 'connect', 742 slave.slave.slavename]) 743 # TODO: add an HTMLLogFile of the exception 744 self.fireTestEvent('attach', why)

745

746 - def detached(self, slave):

747 """This is called when the connection to the bot is lost.""" 748 for sb in self.attaching_slaves + self.slaves: 749 if sb.slave == slave: 750 break 751 else: 752 log.msg("WEIRD: Builder.detached(%s) (%s)" 753 " not in attaching_slaves(%s)" 754 " or slaves(%s)" % (slave, slave.slavename, 755 self.attaching_slaves, 756 self.slaves)) 757 return 758 if sb.state == BUILDING: 759 # the Build's .lostRemote method (invoked by a notifyOnDisconnect 760 # handler) will cause the Build to be stopped, probably right 761 # after the notifyOnDisconnect that invoked us finishes running. 762 763 # TODO: should failover to a new Build 764 #self.retryBuild(sb.build) 765 pass 766 767 if sb in self.attaching_slaves: 768 self.attaching_slaves.remove(sb) 769 if sb in self.slaves: 770 self.slaves.remove(sb) 771 772 # TODO: make this .addSlaveEvent? 773 self.builder_status.addPointEvent(['disconnect', slave.slavename]) 774 sb.detached() # inform the SlaveBuilder that their slave went away 775 self.updateBigStatus() 776 self.fireTestEvent('detach') 777 if not self.slaves: 778 self.fireTestEvent('detach_all')

779

780 - def updateBigStatus(self):

781 if not self.slaves: 782 self.builder_status.setBigState("offline") 783 elif self.building: 784 self.builder_status.setBigState("building") 785 else: 786 self.builder_status.setBigState("idle") 787 self.fireTestEvent('idle')

788

789 - def startBuild(self, build, sb):

790 """Start a build on the given slave. 791 @param build: the L{base.Build} to start 792 @param sb: the L{SlaveBuilder} which will host this build 793 794 @return: a Deferred which fires with a 795 L{buildbot.interfaces.IBuildControl} that can be used to stop the 796 Build, or to access a L{buildbot.interfaces.IBuildStatus} which will 797 watch the Build as it runs. """ 798 799 self.building.append(build) 800 self.updateBigStatus() 801 log.msg("starting build %s using slave %s" % (build, sb)) 802 d = sb.prepare(self.builder_status) 803 def _ping(ign): 804 # ping the slave to make sure they're still there. If they've 805 # fallen off the map (due to a NAT timeout or something), this 806 # will fail in a couple of minutes, depending upon the TCP 807 # timeout. 808 # 809 # TODO: This can unnecessarily suspend the starting of a build, in 810 # situations where the slave is live but is pushing lots of data to 811 # us in a build. 812 log.msg("starting build %s.. pinging the slave %s" % (build, sb)) 813 return sb.ping()

814 d.addCallback(_ping) 815 d.addCallback(self._startBuild_1, build, sb) 816 return d

817

818 - def _startBuild_1(self, res, build, sb):

819 if not res: 820 return self._startBuildFailed("slave ping failed", build, sb) 821 # The buildslave is ready to go. sb.buildStarted() sets its state to 822 # BUILDING (so we won't try to use it for any other builds). This 823 # gets set back to IDLE by the Build itself when it finishes. 824 sb.buildStarted() 825 d = sb.remote.callRemote("startBuild") 826 d.addCallbacks(self._startBuild_2, self._startBuildFailed, 827 callbackArgs=(build,sb), errbackArgs=(build,sb)) 828 return d

829

830 - def _startBuild_2(self, res, build, sb):

831 # create the BuildStatus object that goes with the Build 832 bs = self.builder_status.newBuild() 833 834 # start the build. This will first set up the steps, then tell the 835 # BuildStatus that it has started, which will announce it to the 836 # world (through our BuilderStatus object, which is its parent). 837 # Finally it will start the actual build process. 838 bids = [self.db.build_started(req.id, bs.number) for req in build.requests] 839 d = build.startBuild(bs, self.expectations, sb) 840 d.addCallback(self.buildFinished, sb, bids) 841 # this shouldn't happen. if it does, the slave will be wedged 842 d.addErrback(log.err) 843 return build # this is the IBuildControl

844

845 - def _startBuildFailed(self, why, build, sb):

846 # put the build back on the buildable list 847 log.msg("I tried to tell the slave that the build %s started, but " 848 "remote_startBuild failed: %s" % (build, why)) 849 # release the slave. This will queue a call to maybeStartBuild, which 850 # will fire after other notifyOnDisconnect handlers have marked the 851 # slave as disconnected (so we don't try to use it again). 852 sb.buildFinished() 853 854 log.msg("re-queueing the BuildRequest") 855 self.building.remove(build) 856 self._resubmit_buildreqs(build).addErrback(log.err)

857

858 - def setupProperties(self, props):

859 props.setProperty("buildername", self.name, "Builder") 860 if len(self.properties) > 0: 861 for propertyname in self.properties: 862 props.setProperty(propertyname, self.properties[propertyname], "Builder")

863

864 - def buildFinished(self, build, sb, bids):

865 """This is called when the Build has finished (either success or 866 failure). Any exceptions during the build are reported with 867 results=FAILURE, not with an errback.""" 868 869 # by the time we get here, the Build has already released the slave 870 # (which queues a call to maybeStartBuild) 871 872 self.db.builds_finished(bids) 873 874 results = build.build_status.getResults() 875 self.building.remove(build) 876 if results == RETRY: 877 self._resubmit_buildreqs(build).addErrback(log.err) # returns Deferred 878 else: 879 brids = [br.id for br in build.requests] 880 self.db.retire_buildrequests(brids, results) 881 self.triggerNewBuildCheck()

882

883 - def _resubmit_buildreqs(self, build):

884 brids = [br.id for br in build.requests] 885 return self.db.resubmit_buildrequests(brids)

886

887 - def setExpectations(self, progress):

888 """Mark the build as successful and update expectations for the next 889 build. Only call this when the build did not fail in any way that 890 would invalidate the time expectations generated by it. (if the 891 compile failed and thus terminated early, we can't use the last 892 build to predict how long the next one will take). 893 """ 894 if self.expectations: 895 self.expectations.update(progress) 896 else: 897 # the first time we get a good build, create our Expectations 898 # based upon its results 899 self.expectations = Expectations(progress) 900 log.msg("new expectations: %s seconds" % \ 901 self.expectations.expectedBuildTime())

902

903 - def shutdownSlave(self):

904 if self.remote: 905 self.remote.callRemote("shutdown")

906 907

908 -class BuilderControl:

909 implements(interfaces.IBuilderControl) 910

911 - def __init__(self, builder, parent):

912 self.original = builder 913 self.parent = parent # the IControl object

914

915 - def submitBuildRequest(self, ss, reason, props=None, now=False):

916 bss = self.parent.submitBuildSet([self.original.name], ss, reason, 917 props, now) 918 brs = bss.getBuildRequests()[0] 919 return brs

920

921 - def rebuildBuild(self, bs, reason="<rebuild, no reason given>", extraProperties=None):

922 if not bs.isFinished(): 923 return 924 925 ss = bs.getSourceStamp(absolute=True) 926 if extraProperties is None: 927 properties = bs.getProperties() 928 else: 929 # Make a copy so as not to modify the original build. 930 properties = Properties() 931 properties.updateFromProperties(bs.getProperties()) 932 properties.updateFromProperties(extraProperties) 933 self.submitBuildRequest(ss, reason, props=properties)

934

935 - def getPendingBuilds(self):

936 # return IBuildRequestControl objects 937 retval = [] 938 for r in self.original.getBuildable(): 939 retval.append(BuildRequestControl(self.original, r)) 940 941 return retval

942

943 - def getBuild(self, number):

944 return self.original.getBuild(number)

945

946 - def ping(self):

947 if not self.original.slaves: 948 self.original.builder_status.addPointEvent(["ping", "no slave"]) 949 return defer.succeed(False) # interfaces.NoSlaveError 950 dl = [] 951 for s in self.original.slaves: 952 dl.append(s.ping(self.original.builder_status)) 953 d = defer.DeferredList(dl) 954 d.addCallback(self._gatherPingResults) 955 return d

956

957 - def _gatherPingResults(self, res):

958 for ignored,success in res: 959 if not success: 960 return False 961 return True

962

963 -class BuildRequestControl:

964 implements(interfaces.IBuildRequestControl) 965

966 - def __init__(self, builder, request):

967 self.original_builder = builder 968 self.original_request = request 969 self.brid = request.id

970 973

974 - def unsubscribe(self, observer):

975 raise NotImplementedError

976

977 - def cancel(self):

978 self.original_builder.cancelBuildRequest(self.brid)

979

Source Code for Module buildbot.process.builder