buildbot.process.builder

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 17 import random, weakref 18 from zope.interface import implements 19 from twisted.python import log, failure 20 from twisted.spread import pb 21 from twisted.application import service, internet 22 from twisted.internet import defer 23 24 from buildbot import interfaces 25 from buildbot.status.progress import Expectations 26 from buildbot.status.builder import RETRY 27 from buildbot.status.buildrequest import BuildRequestStatus 28 from buildbot.process.properties import Properties 29 from buildbot.process import buildrequest, slavebuilder 30 from buildbot.process.slavebuilder import BUILDING 31 from buildbot.db import buildrequests

32 33 -class Builder(pb.Referenceable, service.MultiService):

34 """I manage all Builds of a given type. 35 36 Each Builder is created by an entry in the config file (the c['builders'] 37 list), with a number of parameters. 38 39 One of these parameters is the L{buildbot.process.factory.BuildFactory} 40 object that is associated with this Builder. The factory is responsible 41 for creating new L{Build<buildbot.process.build.Build>} objects. Each 42 Build object defines when and how the build is performed, so a new 43 Factory or Builder should be defined to control this behavior. 44 45 The Builder holds on to a number of L{BuildRequest} objects in a 46 list named C{.buildable}. Incoming BuildRequest objects will be added to 47 this list, or (if possible) merged into an existing request. When a slave 48 becomes available, I will use my C{BuildFactory} to turn the request into 49 a new C{Build} object. The C{BuildRequest} is forgotten, the C{Build} 50 goes into C{.building} while it runs. Once the build finishes, I will 51 discard it. 52 53 I maintain a list of available SlaveBuilders, one for each connected 54 slave that the C{slavenames} parameter says we can use. Some of these 55 will be idle, some of them will be busy running builds for me. If there 56 are multiple slaves, I can run multiple builds at once. 57 58 I also manage forced builds, progress expectation (ETA) management, and 59 some status delivery chores. 60 61 @type buildable: list of L{buildbot.process.buildrequest.BuildRequest} 62 @ivar buildable: BuildRequests that are ready to build, but which are 63 waiting for a buildslave to be available. 64 65 @type building: list of L{buildbot.process.build.Build} 66 @ivar building: Builds that are actively running 67 68 @type slaves: list of L{buildbot.buildslave.BuildSlave} objects 69 @ivar slaves: the slaves currently available for building 70 """ 71 72 expectations = None # this is created the first time we get a good build 73

74 - def __init__(self, setup, builder_status):

75 """ 76 @type setup: dict 77 @param setup: builder setup data, as stored in 78 BuildmasterConfig['builders']. Contains name, 79 slavename(s), builddir, slavebuilddir, factory, locks. 80 @type builder_status: L{buildbot.status.builder.BuilderStatus} 81 """ 82 service.MultiService.__init__(self) 83 self.name = setup['name'] 84 self.slavenames = [] 85 if setup.has_key('slavename'): 86 self.slavenames.append(setup['slavename']) 87 if setup.has_key('slavenames'): 88 self.slavenames.extend(setup['slavenames']) 89 self.builddir = setup['builddir'] 90 self.slavebuilddir = setup['slavebuilddir'] 91 self.buildFactory = setup['factory'] 92 self.nextSlave = setup.get('nextSlave') 93 if self.nextSlave is not None and not callable(self.nextSlave): 94 raise ValueError("nextSlave must be callable") 95 self.locks = setup.get("locks", []) 96 self.env = setup.get('env', {}) 97 assert isinstance(self.env, dict) 98 if setup.has_key('periodicBuildTime'): 99 raise ValueError("periodicBuildTime can no longer be defined as" 100 " part of the Builder: use scheduler.Periodic" 101 " instead") 102 self.nextBuild = setup.get('nextBuild') 103 if self.nextBuild is not None and not callable(self.nextBuild): 104 raise ValueError("nextBuild must be callable") 105 self.buildHorizon = setup.get('buildHorizon') 106 self.logHorizon = setup.get('logHorizon') 107 self.eventHorizon = setup.get('eventHorizon') 108 self.mergeRequests = setup.get('mergeRequests', None) 109 self.properties = setup.get('properties', {}) 110 self.category = setup.get('category', None) 111 112 # build/wannabuild slots: Build objects move along this sequence 113 self.building = [] 114 # old_building holds active builds that were stolen from a predecessor 115 self.old_building = weakref.WeakKeyDictionary() 116 117 # buildslaves which have connected but which are not yet available. 118 # These are always in the ATTACHING state. 119 self.attaching_slaves = [] 120 121 # buildslaves at our disposal. Each SlaveBuilder instance has a 122 # .state that is IDLE, PINGING, or BUILDING. "PINGING" is used when a 123 # Build is about to start, to make sure that they're still alive. 124 self.slaves = [] 125 126 self.builder_status = builder_status 127 self.builder_status.setSlavenames(self.slavenames) 128 self.builder_status.buildHorizon = self.buildHorizon 129 self.builder_status.logHorizon = self.logHorizon 130 self.builder_status.eventHorizon = self.eventHorizon 131 132 self.reclaim_svc = internet.TimerService(10*60, self.reclaimAllBuilds) 133 self.reclaim_svc.setServiceParent(self) 134 135 # for testing, to help synchronize tests 136 self.run_count = 0

137

138 - def stopService(self):

139 d = defer.maybeDeferred(lambda : 140 service.MultiService.stopService(self)) 141 def flushMaybeStartBuilds(_): 142 # at this point, self.running = False, so another maybeStartBuilds 143 # invocation won't hurt anything, but it also will not complete 144 # until any currently-running invocations are done. 145 return self.maybeStartBuild()

146 d.addCallback(flushMaybeStartBuilds) 147 return d

148

149 - def setBotmaster(self, botmaster):

150 self.botmaster = botmaster 151 self.master = botmaster.master 152 self.db = self.master.db

153

154 - def compareToSetup(self, setup):

155 diffs = [] 156 setup_slavenames = [] 157 if setup.has_key('slavename'): 158 setup_slavenames.append(setup['slavename']) 159 setup_slavenames.extend(setup.get('slavenames', [])) 160 if setup_slavenames != self.slavenames: 161 diffs.append('slavenames changed from %s to %s' \ 162 % (self.slavenames, setup_slavenames)) 163 if setup['builddir'] != self.builddir: 164 diffs.append('builddir changed from %s to %s' \ 165 % (self.builddir, setup['builddir'])) 166 if setup['slavebuilddir'] != self.slavebuilddir: 167 diffs.append('slavebuilddir changed from %s to %s' \ 168 % (self.slavebuilddir, setup['slavebuilddir'])) 169 if setup['factory'] != self.buildFactory: # compare objects 170 diffs.append('factory changed') 171 if setup.get('locks', []) != self.locks: 172 diffs.append('locks changed from %s to %s' % (self.locks, setup.get('locks'))) 173 if setup.get('env', {}) != self.env: 174 diffs.append('env changed from %s to %s' % (self.env, setup.get('env', {}))) 175 if setup.get('nextSlave') != self.nextSlave: 176 diffs.append('nextSlave changed from %s to %s' % (self.nextSlave, setup.get('nextSlave'))) 177 if setup.get('nextBuild') != self.nextBuild: 178 diffs.append('nextBuild changed from %s to %s' % (self.nextBuild, setup.get('nextBuild'))) 179 if setup.get('buildHorizon', None) != self.buildHorizon: 180 diffs.append('buildHorizon changed from %s to %s' % (self.buildHorizon, setup['buildHorizon'])) 181 if setup.get('logHorizon', None) != self.logHorizon: 182 diffs.append('logHorizon changed from %s to %s' % (self.logHorizon, setup['logHorizon'])) 183 if setup.get('eventHorizon', None) != self.eventHorizon: 184 diffs.append('eventHorizon changed from %s to %s' % (self.eventHorizon, setup['eventHorizon'])) 185 if setup.get('category', None) != self.category: 186 diffs.append('category changed from %r to %r' % (self.category, setup.get('category', None))) 187 188 return diffs

189

190 - def __repr__(self):

191 return "<Builder '%r' at %d>" % (self.name, id(self))

192 193 @defer.deferredGenerator

194 - def getOldestRequestTime(self):

195 196 """Returns the submitted_at of the oldest unclaimed build request for 197 this builder, or None if there are no build requests. 198 199 @returns: datetime instance or None, via Deferred 200 """ 201 wfd = defer.waitForDeferred( 202 self.master.db.buildrequests.getBuildRequests( 203 buildername=self.name, claimed=False)) 204 yield wfd 205 unclaimed = wfd.getResult() 206 207 if unclaimed: 208 unclaimed = [ brd['submitted_at'] for brd in unclaimed ] 209 unclaimed.sort() 210 yield unclaimed[0] 211 else: 212 yield None

213

214 - def consumeTheSoulOfYourPredecessor(self, old):

215 """Suck the brain out of an old Builder. 216 217 This takes all the runtime state from an existing Builder and moves 218 it into ourselves. This is used when a Builder is changed in the 219 master.cfg file: the new Builder has a different factory, but we want 220 all the builds that were queued for the old one to get processed by 221 the new one. Any builds which are already running will keep running. 222 The new Builder will get as many of the old SlaveBuilder objects as 223 it wants.""" 224 225 log.msg("consumeTheSoulOfYourPredecessor: %s feeding upon %s" % 226 (self, old)) 227 # all pending builds are stored in the DB, so we don't have to do 228 # anything to claim them. The old builder will be stopService'd, 229 # which should make sure they don't start any new work 230 231 # this is kind of silly, but the builder status doesn't get updated 232 # when the config changes, yet it stores the category. So: 233 self.builder_status.category = self.category 234 235 # old.building (i.e. builds which are still running) is not migrated 236 # directly: it keeps track of builds which were in progress in the 237 # old Builder. When those builds finish, the old Builder will be 238 # notified, not us. However, since the old SlaveBuilder will point to 239 # us, it is our maybeStartBuild() that will be triggered. 240 if old.building: 241 self.builder_status.setBigState("building") 242 # however, we do grab a weakref to the active builds, so that our 243 # BuilderControl can see them and stop them. We use a weakref because 244 # we aren't the one to get notified, so there isn't a convenient 245 # place to remove it from self.building . 246 for b in old.building: 247 self.old_building[b] = None 248 for b in old.old_building: 249 self.old_building[b] = None 250 251 # Our set of slavenames may be different. Steal any of the old 252 # buildslaves that we want to keep using. 253 for sb in old.slaves[:]: 254 if sb.slave.slavename in self.slavenames: 255 log.msg(" stealing buildslave %s" % sb) 256 self.slaves.append(sb) 257 old.slaves.remove(sb) 258 sb.setBuilder(self) 259 260 # old.attaching_slaves: 261 # these SlaveBuilders are waiting on a sequence of calls: 262 # remote.setMaster and remote.print . When these two complete, 263 # old._attached will be fired, which will add a 'connect' event to 264 # the builder_status and try to start a build. However, we've pulled 265 # everything out of the old builder's queue, so it will have no work 266 # to do. The outstanding remote.setMaster/print call will be holding 267 # the last reference to the old builder, so it will disappear just 268 # after that response comes back. 269 # 270 # The BotMaster will ask the slave to re-set their list of Builders 271 # shortly after this function returns, which will cause our 272 # attached() method to be fired with a bunch of references to remote 273 # SlaveBuilders, some of which we already have (by stealing them 274 # from the old Builder), some of which will be new. The new ones 275 # will be re-attached. 276 277 # Therefore, we don't need to do anything about old.attaching_slaves 278 279 return # all done

280

281 - def reclaimAllBuilds(self):

282 brids = set() 283 for b in self.building: 284 brids.update([br.id for br in b.requests]) 285 for b in self.old_building: 286 brids.update([br.id for br in b.requests]) 287 288 if not brids: 289 return defer.succeed(None) 290 291 d = self.master.db.buildrequests.reclaimBuildRequests(brids) 292 d.addErrback(log.err, 'while re-claiming running BuildRequests') 293 return d

294

295 - def getBuild(self, number):

296 for b in self.building: 297 if b.build_status and b.build_status.number == number: 298 return b 299 for b in self.old_building.keys(): 300 if b.build_status and b.build_status.number == number: 301 return b 302 return None

303

304 - def addLatentSlave(self, slave):

305 assert interfaces.ILatentBuildSlave.providedBy(slave) 306 for s in self.slaves: 307 if s == slave: 308 break 309 else: 310 sb = slavebuilder.LatentSlaveBuilder(slave, self) 311 self.builder_status.addPointEvent( 312 ['added', 'latent', slave.slavename]) 313 self.slaves.append(sb) 314 self.botmaster.maybeStartBuildsForBuilder(self.name)

315

316 - def attached(self, slave, remote, commands):

317 """This is invoked by the BuildSlave when the self.slavename bot 318 registers their builder. 319 320 @type slave: L{buildbot.buildslave.BuildSlave} 321 @param slave: the BuildSlave that represents the buildslave as a whole 322 @type remote: L{twisted.spread.pb.RemoteReference} 323 @param remote: a reference to the L{buildbot.slave.bot.SlaveBuilder} 324 @type commands: dict: string -> string, or None 325 @param commands: provides the slave's version of each RemoteCommand 326 327 @rtype: L{twisted.internet.defer.Deferred} 328 @return: a Deferred that fires (with 'self') when the slave-side 329 builder is fully attached and ready to accept commands. 330 """ 331 for s in self.attaching_slaves + self.slaves: 332 if s.slave == slave: 333 # already attached to them. This is fairly common, since 334 # attached() gets called each time we receive the builder 335 # list from the slave, and we ask for it each time we add or 336 # remove a builder. So if the slave is hosting builders 337 # A,B,C, and the config file changes A, we'll remove A and 338 # re-add it, triggering two builder-list requests, getting 339 # two redundant calls to attached() for B, and another two 340 # for C. 341 # 342 # Therefore, when we see that we're already attached, we can 343 # just ignore it. 344 return defer.succeed(self) 345 346 sb = slavebuilder.SlaveBuilder() 347 sb.setBuilder(self) 348 self.attaching_slaves.append(sb) 349 d = sb.attached(slave, remote, commands) 350 d.addCallback(self._attached) 351 d.addErrback(self._not_attached, slave) 352 return d

353

354 - def _attached(self, sb):

355 self.builder_status.addPointEvent(['connect', sb.slave.slavename]) 356 self.attaching_slaves.remove(sb) 357 self.slaves.append(sb) 358 359 self.updateBigStatus() 360 361 return self

362

363 - def _not_attached(self, why, slave):

364 # already log.err'ed by SlaveBuilder._attachFailure 365 # TODO: remove from self.slaves (except that detached() should get 366 # run first, right?) 367 log.err(why, 'slave failed to attach') 368 self.builder_status.addPointEvent(['failed', 'connect', 369 slave.slavename])

370 # TODO: add an HTMLLogFile of the exception 371

372 - def detached(self, slave):

373 """This is called when the connection to the bot is lost.""" 374 for sb in self.attaching_slaves + self.slaves: 375 if sb.slave == slave: 376 break 377 else: 378 log.msg("WEIRD: Builder.detached(%s) (%s)" 379 " not in attaching_slaves(%s)" 380 " or slaves(%s)" % (slave, slave.slavename, 381 self.attaching_slaves, 382 self.slaves)) 383 return 384 if sb.state == BUILDING: 385 # the Build's .lostRemote method (invoked by a notifyOnDisconnect 386 # handler) will cause the Build to be stopped, probably right 387 # after the notifyOnDisconnect that invoked us finishes running. 388 pass 389 390 if sb in self.attaching_slaves: 391 self.attaching_slaves.remove(sb) 392 if sb in self.slaves: 393 self.slaves.remove(sb) 394 395 self.builder_status.addPointEvent(['disconnect', slave.slavename]) 396 sb.detached() # inform the SlaveBuilder that their slave went away 397 self.updateBigStatus()

398

399 - def updateBigStatus(self):

400 if not self.slaves: 401 self.builder_status.setBigState("offline") 402 elif self.building: 403 self.builder_status.setBigState("building") 404 else: 405 self.builder_status.setBigState("idle")

406 407 @defer.deferredGenerator

408 - def _startBuildFor(self, slavebuilder, buildrequests):

409 """Start a build on the given slave. 410 @param build: the L{base.Build} to start 411 @param sb: the L{SlaveBuilder} which will host this build 412 413 @return: (via Deferred) boolean indicating that the build was 414 succesfully started. 415 """ 416 417 # as of the Python versions supported now, try/finally can't be used 418 # with a generator expression. So instead, we push cleanup functions 419 # into a list so that, at any point, we can abort this operation. 420 cleanups = [] 421 def run_cleanups(): 422 while cleanups: 423 fn = cleanups.pop() 424 fn()

425 426 # the last cleanup we want to perform is to update the big 427 # status based on any other cleanup 428 cleanups.append(lambda : self.updateBigStatus()) 429 430 build = self.buildFactory.newBuild(buildrequests) 431 build.setBuilder(self) 432 log.msg("starting build %s using slave %s" % (build, slavebuilder)) 433 434 # set up locks 435 build.setLocks(self.locks) 436 cleanups.append(lambda : slavebuilder.slave.releaseLocks()) 437 438 if len(self.env) > 0: 439 build.setSlaveEnvironment(self.env) 440 441 # append the build to self.building 442 self.building.append(build) 443 cleanups.append(lambda : self.building.remove(build)) 444 445 # update the big status accordingly 446 self.updateBigStatus() 447 448 try: 449 wfd = defer.waitForDeferred( 450 slavebuilder.prepare(self.builder_status, build)) 451 yield wfd 452 ready = wfd.getResult() 453 except: 454 log.err(failure.Failure(), 'while preparing slavebuilder:') 455 ready = False 456 457 # If prepare returns True then it is ready and we start a build 458 # If it returns false then we don't start a new build. 459 if not ready: 460 log.msg("slave %s can't build %s after all; re-queueing the " 461 "request" % (build, slavebuilder)) 462 run_cleanups() 463 yield False 464 return 465 466 # ping the slave to make sure they're still there. If they've 467 # fallen off the map (due to a NAT timeout or something), this 468 # will fail in a couple of minutes, depending upon the TCP 469 # timeout. 470 # 471 # TODO: This can unnecessarily suspend the starting of a build, in 472 # situations where the slave is live but is pushing lots of data to 473 # us in a build. 474 log.msg("starting build %s.. pinging the slave %s" 475 % (build, slavebuilder)) 476 try: 477 wfd = defer.waitForDeferred( 478 slavebuilder.ping()) 479 yield wfd 480 ping_success = wfd.getResult() 481 except: 482 log.err(failure.Failure(), 'while pinging slave before build:') 483 ping_success = False 484 485 if not ping_success: 486 log.msg("slave ping failed; re-queueing the request") 487 run_cleanups() 488 yield False 489 return 490 491 # The buildslave is ready to go. slavebuilder.buildStarted() sets its 492 # state to BUILDING (so we won't try to use it for any other builds). 493 # This gets set back to IDLE by the Build itself when it finishes. 494 slavebuilder.buildStarted() 495 cleanups.append(lambda : slavebuilder.buildFinished()) 496 497 # tell the remote that it's starting a build, too 498 try: 499 wfd = defer.waitForDeferred( 500 slavebuilder.remote.callRemote("startBuild")) 501 yield wfd 502 wfd.getResult() 503 except: 504 log.err(failure.Failure(), 'while calling remote startBuild:') 505 run_cleanups() 506 yield False 507 return 508 509 # create the BuildStatus object that goes with the Build 510 bs = self.builder_status.newBuild() 511 512 # record the build in the db - one row per buildrequest 513 try: 514 bids = [] 515 for req in build.requests: 516 wfd = defer.waitForDeferred( 517 self.master.db.builds.addBuild(req.id, bs.number)) 518 yield wfd 519 bids.append(wfd.getResult()) 520 except: 521 log.err(failure.Failure(), 'while adding rows to build table:') 522 run_cleanups() 523 yield False 524 return 525 526 # let status know 527 self.master.status.build_started(req.id, self.name, bs) 528 529 # start the build. This will first set up the steps, then tell the 530 # BuildStatus that it has started, which will announce it to the world 531 # (through our BuilderStatus object, which is its parent). Finally it 532 # will start the actual build process. This is done with a fresh 533 # Deferred since _startBuildFor should not wait until the build is 534 # finished. 535 d = build.startBuild(bs, self.expectations, slavebuilder) 536 d.addCallback(self.buildFinished, slavebuilder, bids) 537 # this shouldn't happen. if it does, the slave will be wedged 538 d.addErrback(log.err) 539 540 # make sure the builder's status is represented correctly 541 self.updateBigStatus() 542 543 yield True 544

545 - def setupProperties(self, props):

546 props.setProperty("buildername", self.name, "Builder") 547 if len(self.properties) > 0: 548 for propertyname in self.properties: 549 props.setProperty(propertyname, self.properties[propertyname], 550 "Builder")

551

552 - def buildFinished(self, build, sb, bids):

553 """This is called when the Build has finished (either success or 554 failure). Any exceptions during the build are reported with 555 results=FAILURE, not with an errback.""" 556 557 # by the time we get here, the Build has already released the slave, 558 # which will trigger a check for any now-possible build requests 559 # (maybeStartBuilds) 560 561 # mark the builds as finished, although since nothing ever reads this 562 # table, it's not too important that it complete successfully 563 d = self.db.builds.finishBuilds(bids) 564 d.addErrback(log.err, 'while marking builds as finished (ignored)') 565 566 results = build.build_status.getResults() 567 self.building.remove(build) 568 if results == RETRY: 569 self._resubmit_buildreqs(build).addErrback(log.err) 570 else: 571 brids = [br.id for br in build.requests] 572 db = self.master.db 573 d = db.buildrequests.completeBuildRequests(brids, results) 574 d.addCallback( 575 lambda _ : self._maybeBuildsetsComplete(build.requests)) 576 # nothing in particular to do with this deferred, so just log it if 577 # it fails.. 578 d.addErrback(log.err, 'while marking build requests as completed') 579 580 if sb.slave: 581 sb.slave.releaseLocks() 582 583 self.updateBigStatus()

584 585 @defer.deferredGenerator

586 - def _maybeBuildsetsComplete(self, requests):

587 # inform the master that we may have completed a number of buildsets 588 for br in requests: 589 wfd = defer.waitForDeferred( 590 self.master.maybeBuildsetComplete(br.bsid)) 591 yield wfd 592 wfd.getResult()

593

594 - def _resubmit_buildreqs(self, build):

595 brids = [br.id for br in build.requests] 596 return self.db.buildrequests.unclaimBuildRequests(brids)

597

598 - def setExpectations(self, progress):

599 """Mark the build as successful and update expectations for the next 600 build. Only call this when the build did not fail in any way that 601 would invalidate the time expectations generated by it. (if the 602 compile failed and thus terminated early, we can't use the last 603 build to predict how long the next one will take). 604 """ 605 if self.expectations: 606 self.expectations.update(progress) 607 else: 608 # the first time we get a good build, create our Expectations 609 # based upon its results 610 self.expectations = Expectations(progress) 611 log.msg("new expectations: %s seconds" % \ 612 self.expectations.expectedBuildTime())

613 614 # Build Creation 615 616 @defer.deferredGenerator

617 - def maybeStartBuild(self):

618 # This method is called by the botmaster whenever this builder should 619 # check for and potentially start new builds. Do not call this method 620 # directly - use master.botmaster.maybeStartBuildsForBuilder, or one 621 # of the other similar methods if more appropriate 622 623 # first, if we're not running, then don't start builds; stopService 624 # uses this to ensure that any ongoing maybeStartBuild invocations 625 # are complete before it stops. 626 if not self.running: 627 return 628 629 # Check for available slaves. If there are no available slaves, then 630 # there is no sense continuing 631 available_slavebuilders = [ sb for sb in self.slaves 632 if sb.isAvailable() ] 633 if not available_slavebuilders: 634 self.updateBigStatus() 635 return 636 637 # now, get the available build requests 638 wfd = defer.waitForDeferred( 639 self.master.db.buildrequests.getBuildRequests( 640 buildername=self.name, claimed=False)) 641 yield wfd 642 unclaimed_requests = wfd.getResult() 643 644 if not unclaimed_requests: 645 self.updateBigStatus() 646 return 647 648 # sort by submitted_at, so the first is the oldest 649 unclaimed_requests.sort(key=lambda brd : brd['submitted_at']) 650 651 # get the mergeRequests function for later 652 mergeRequests_fn = self._getMergeRequestsFn() 653 654 # match them up until we're out of options 655 while available_slavebuilders and unclaimed_requests: 656 # first, choose a slave (using nextSlave) 657 wfd = defer.waitForDeferred( 658 self._chooseSlave(available_slavebuilders)) 659 yield wfd 660 slavebuilder = wfd.getResult() 661 662 if not slavebuilder: 663 break 664 665 if slavebuilder not in available_slavebuilders: 666 log.msg(("nextSlave chose a nonexistent slave for builder " 667 "'%s'; cannot start build") % self.name) 668 break 669 670 # then choose a request (using nextBuild) 671 wfd = defer.waitForDeferred( 672 self._chooseBuild(unclaimed_requests)) 673 yield wfd 674 brdict = wfd.getResult() 675 676 if not brdict: 677 break 678 679 if brdict not in unclaimed_requests: 680 log.msg(("nextBuild chose a nonexistent request for builder " 681 "'%s'; cannot start build") % self.name) 682 break 683 684 # merge the chosen request with any compatible requests in the 685 # queue 686 wfd = defer.waitForDeferred( 687 self._mergeRequests(brdict, unclaimed_requests, 688 mergeRequests_fn)) 689 yield wfd 690 brdicts = wfd.getResult() 691 692 # try to claim the build requests 693 brids = [ brdict['brid'] for brdict in brdicts ] 694 try: 695 wfd = defer.waitForDeferred( 696 self.master.db.buildrequests.claimBuildRequests(brids)) 697 yield wfd 698 wfd.getResult() 699 except buildrequests.AlreadyClaimedError: 700 # one or more of the build requests was already claimed; 701 # re-fetch the now-partially-claimed build requests and keep 702 # trying to match them 703 self._breakBrdictRefloops(unclaimed_requests) 704 wfd = defer.waitForDeferred( 705 self.master.db.buildrequests.getBuildRequests( 706 buildername=self.name, claimed=False)) 707 yield wfd 708 unclaimed_requests = wfd.getResult() 709 710 # go around the loop again 711 continue 712 713 # claim was successful, so initiate a build for this set of 714 # requests. Note that if the build fails from here on out (e.g., 715 # because a slave has failed), it will be handled outside of this 716 # loop. TODO: test that! 717 718 # _startBuildFor expects BuildRequest objects, so cook some up 719 wfd = defer.waitForDeferred( 720 defer.gatherResults([ self._brdictToBuildRequest(brdict) 721 for brdict in brdicts ])) 722 yield wfd 723 breqs = wfd.getResult() 724 725 wfd = defer.waitForDeferred( 726 self._startBuildFor(slavebuilder, breqs)) 727 yield wfd 728 build_started = wfd.getResult() 729 730 if not build_started: 731 # build was not started, so unclaim the build requests 732 wfd = defer.waitForDeferred( 733 self.master.db.buildrequests.unclaimBuildRequests(brids)) 734 yield wfd 735 wfd.getResult() 736 737 # and try starting builds again. If we still have a working slave, 738 # then this may re-claim the same buildrequests 739 self.botmaster.maybeStartBuildsForBuilder(self.name) 740 741 # finally, remove the buildrequests and slavebuilder from the 742 # respective queues 743 self._breakBrdictRefloops(brdicts) 744 for brdict in brdicts: 745 unclaimed_requests.remove(brdict) 746 available_slavebuilders.remove(slavebuilder) 747 748 self._breakBrdictRefloops(unclaimed_requests) 749 self.updateBigStatus() 750 return

751 752 # a few utility functions to make the maybeStartBuild a bit shorter and 753 # easier to read 754

755 - def _chooseSlave(self, available_slavebuilders):

756 """ 757 Choose the next slave, using the C{nextSlave} configuration if 758 available, and falling back to C{random.choice} otherwise. 759 760 @param available_slavebuilders: list of slavebuilders to choose from 761 @returns: SlaveBuilder or None via Deferred 762 """ 763 if self.nextSlave: 764 return defer.maybeDeferred(lambda : 765 self.nextSlave(self, available_slavebuilders)) 766 else: 767 return defer.succeed(random.choice(available_slavebuilders))

768

769 - def _chooseBuild(self, buildrequests):

770 """ 771 Choose the next build from the given set of build requests (represented 772 as dictionaries). Defaults to returning the first request (earliest 773 submitted). 774 775 @param buildrequests: sorted list of build request dictionaries 776 @returns: a build request dictionary or None via Deferred 777 """ 778 if self.nextBuild: 779 # nextBuild expects BuildRequest objects, so instantiate them here 780 # and cache them in the dictionaries 781 d = defer.gatherResults([ self._brdictToBuildRequest(brdict) 782 for brdict in buildrequests ]) 783 d.addCallback(lambda requestobjects : 784 self.nextBuild(self, requestobjects)) 785 def to_brdict(brobj): 786 # get the brdict for this object back 787 return brobj.brdict

788 d.addCallback(to_brdict) 789 return d 790 else: 791 return defer.succeed(buildrequests[0]) 792

793 - def _getMergeRequestsFn(self):

794 """Helper function to determine which mergeRequests function to use 795 from L{_mergeRequests}, or None for no merging""" 796 # first, seek through builder, global, and the default 797 mergeRequests_fn = self.mergeRequests 798 if mergeRequests_fn is None: 799 mergeRequests_fn = self.botmaster.mergeRequests 800 if mergeRequests_fn is None: 801 mergeRequests_fn = True 802 803 # then translate False and True properly 804 if mergeRequests_fn is False: 805 mergeRequests_fn = None 806 elif mergeRequests_fn is True: 807 mergeRequests_fn = Builder._defaultMergeRequestFn 808 809 return mergeRequests_fn

810

811 - def _defaultMergeRequestFn(self, req1, req2):

812 return req1.canBeMergedWith(req2)

813 814 @defer.deferredGenerator

815 - def _mergeRequests(self, breq, unclaimed_requests, mergeRequests_fn):

816 """Use C{mergeRequests_fn} to merge C{breq} against 817 C{unclaimed_requests}, where both are build request dictionaries""" 818 # short circuit if there is no merging to do 819 if not mergeRequests_fn or len(unclaimed_requests) == 1: 820 yield [ breq ] 821 return 822 823 # we'll need BuildRequest objects, so get those first 824 wfd = defer.waitForDeferred( 825 defer.gatherResults( 826 [ self._brdictToBuildRequest(brdict) 827 for brdict in unclaimed_requests ])) 828 yield wfd 829 unclaimed_request_objects = wfd.getResult() 830 breq_object = unclaimed_request_objects.pop( 831 unclaimed_requests.index(breq)) 832 833 # gather the mergeable requests 834 merged_request_objects = [breq_object] 835 for other_breq_object in unclaimed_request_objects: 836 wfd = defer.waitForDeferred( 837 defer.maybeDeferred(lambda : 838 mergeRequests_fn(self, breq_object, other_breq_object))) 839 yield wfd 840 if wfd.getResult(): 841 merged_request_objects.append(other_breq_object) 842 843 # convert them back to brdicts and return 844 merged_requests = [ br.brdict for br in merged_request_objects ] 845 yield merged_requests

846

847 - def _brdictToBuildRequest(self, brdict):

848 """ 849 Convert a build request dictionary to a L{buildrequest.BuildRequest} 850 object, caching the result in the dictionary itself. The resulting 851 buildrequest will have a C{brdict} attribute pointing back to this 852 dictionary. 853 854 Note that this does not perform any locking - be careful that it is 855 only called once at a time for each build request dictionary. 856 857 @param brdict: dictionary to convert 858 859 @returns: L{buildrequest.BuildRequest} via Deferred 860 """ 861 if 'brobj' in brdict: 862 return defer.succeed(brdict['brobj']) 863 d = buildrequest.BuildRequest.fromBrdict(self.master, brdict) 864 def keep(buildrequest): 865 brdict['brobj'] = buildrequest 866 buildrequest.brdict = brdict 867 return buildrequest

868 d.addCallback(keep) 869 return d 870

871 - def _breakBrdictRefloops(self, requests):

872 """Break the reference loops created by L{_brdictToBuildRequest}""" 873 for brdict in requests: 874 try: 875 del brdict['brobj'].brdict 876 except KeyError: 877 pass

878

879 880 -class BuilderControl:

881 implements(interfaces.IBuilderControl) 882

883 - def __init__(self, builder, master):

884 self.original = builder 885 self.master = master

886

887 - def submitBuildRequest(self, ss, reason, props=None):

888 d = ss.getSourceStampId(self.master.master) 889 def add_buildset(ssid): 890 return self.master.master.addBuildset( 891 builderNames=[self.original.name], 892 ssid=ssid, reason=reason, properties=props)

893 d.addCallback(add_buildset) 894 def get_brs((bsid,brids)): 895 brs = BuildRequestStatus(self.original.name, 896 brids[self.original.name], 897 self.master.master.status) 898 return brs

899 d.addCallback(get_brs) 900 return d 901

902 - def rebuildBuild(self, bs, reason="<rebuild, no reason given>", extraProperties=None):

903 if not bs.isFinished(): 904 return 905 906 # Make a copy of the properties so as not to modify the original build. 907 properties = Properties() 908 # Don't include runtime-set properties in a rebuild request 909 properties.updateFromPropertiesNoRuntime(bs.getProperties()) 910 if extraProperties is None: 911 properties.updateFromProperties(extraProperties) 912 913 properties_dict = dict((k,(v,s)) for (k,v,s) in properties.asList()) 914 ss = bs.getSourceStamp(absolute=True) 915 d = ss.getSourceStampId(self.master.master) 916 def add_buildset(ssid): 917 return self.master.master.addBuildset( 918 builderNames=[self.original.name], 919 ssid=ssid, reason=reason, properties=properties_dict)

920 d.addCallback(add_buildset) 921 return d 922 923 @defer.deferredGenerator

924 - def getPendingBuildRequestControls(self):

925 master = self.original.master 926 wfd = defer.waitForDeferred( 927 master.db.buildrequests.getBuildRequests( 928 buildername=self.original.name, 929 claimed=False)) 930 yield wfd 931 brdicts = wfd.getResult() 932 933 # convert those into BuildRequest objects 934 buildrequests = [ ] 935 for brdict in brdicts: 936 wfd = defer.waitForDeferred( 937 buildrequest.BuildRequest.fromBrdict(self.master.master, 938 brdict)) 939 yield wfd 940 buildrequests.append(wfd.getResult()) 941 942 # and return the corresponding control objects 943 yield [ buildrequest.BuildRequestControl(self.original, r) 944 for r in buildrequests ]

945

946 - def getBuild(self, number):

947 return self.original.getBuild(number)

948

949 - def ping(self):

950 if not self.original.slaves: 951 self.original.builder_status.addPointEvent(["ping", "no slave"]) 952 return defer.succeed(False) # interfaces.NoSlaveError 953 dl = [] 954 for s in self.original.slaves: 955 dl.append(s.ping(self.original.builder_status)) 956 d = defer.DeferredList(dl) 957 d.addCallback(self._gatherPingResults) 958 return d

959

960 - def _gatherPingResults(self, res):

961 for ignored,success in res: 962 if not success: 963 return False 964 return True

965

Source Code for Module buildbot.process.builder