buildbot.master

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 17 import os 18 import signal 19 import time 20 import textwrap 21 22 from zope.interface import implements 23 from twisted.python import log, components 24 from twisted.python.failure import Failure 25 from twisted.internet import defer, reactor 26 from twisted.spread import pb 27 from twisted.application import service 28 from twisted.application.internet import TimerService 29 30 import buildbot 31 import buildbot.pbmanager 32 from buildbot.util import now, safeTranslate, eventual 33 from buildbot.pbutil import NewCredPerspective 34 from buildbot.process.builder import Builder, IDLE 35 from buildbot.status.builder import Status, BuildSetStatus 36 from buildbot.changes.changes import Change 37 from buildbot.changes.manager import ChangeManager 38 from buildbot import interfaces, locks 39 from buildbot.process.properties import Properties 40 from buildbot.config import BuilderConfig 41 from buildbot.process.builder import BuilderControl 42 from buildbot.db.dbspec import DBSpec 43 from buildbot.db import connector, exceptions 44 from buildbot.db.schema.manager import DBSchemaManager 45 from buildbot.schedulers.manager import SchedulerManager 46 from buildbot.util.loop import DelegateLoop 47 48 ######################################## 49

50 -class BotMaster(service.MultiService):

51 52 """This is the master-side service which manages remote buildbot slaves. 53 It provides them with BuildSlaves, and distributes file change 54 notification messages to them. 55 """ 56 57 debug = 0 58 reactor = reactor 59

60 - def __init__(self, master):

61 service.MultiService.__init__(self) 62 self.master = master 63 64 self.builders = {} 65 self.builderNames = [] 66 # builders maps Builder names to instances of bb.p.builder.Builder, 67 # which is the master-side object that defines and controls a build. 68 # They are added by calling botmaster.addBuilder() from the startup 69 # code. 70 71 # self.slaves contains a ready BuildSlave instance for each 72 # potential buildslave, i.e. all the ones listed in the config file. 73 # If the slave is connected, self.slaves[slavename].slave will 74 # contain a RemoteReference to their Bot instance. If it is not 75 # connected, that attribute will hold None. 76 self.slaves = {} # maps slavename to BuildSlave 77 self.watchers = {} 78 79 # self.locks holds the real Lock instances 80 self.locks = {} 81 82 # self.mergeRequests is the callable override for merging build 83 # requests 84 self.mergeRequests = None 85 86 # self.prioritizeBuilders is the callable override for builder order 87 # traversal 88 self.prioritizeBuilders = None 89 90 self.loop = DelegateLoop(self._get_processors) 91 self.loop.setServiceParent(self) 92 93 self.shuttingDown = False 94 95 self.lastSlavePortnum = None

96

97 - def setMasterName(self, name, incarnation):

98 self.master_name = name 99 self.master_incarnation = incarnation

100

101 - def cleanShutdown(self):

102 if self.shuttingDown: 103 return 104 log.msg("Initiating clean shutdown") 105 self.shuttingDown = True 106 107 # Wait for all builds to finish 108 l = [] 109 for builder in self.builders.values(): 110 for build in builder.builder_status.getCurrentBuilds(): 111 l.append(build.waitUntilFinished()) 112 if len(l) == 0: 113 log.msg("No running jobs, starting shutdown immediately") 114 self.loop.trigger() 115 d = self.loop.when_quiet() 116 else: 117 log.msg("Waiting for %i build(s) to finish" % len(l)) 118 d = defer.DeferredList(l) 119 d.addCallback(lambda ign: self.loop.when_quiet()) 120 121 # Flush the eventual queue 122 d.addCallback(eventual.flushEventualQueue) 123 124 # Finally, shut the whole process down 125 def shutdown(ign): 126 # Double check that we're still supposed to be shutting down 127 # The shutdown may have been cancelled! 128 if self.shuttingDown: 129 # Check that there really aren't any running builds 130 for builder in self.builders.values(): 131 n = len(builder.builder_status.getCurrentBuilds()) 132 if n > 0: 133 log.msg("Not shutting down, builder %s has %i builds running" % (builder, n)) 134 log.msg("Trying shutdown sequence again") 135 self.shuttingDown = False 136 self.cleanShutdown() 137 return 138 log.msg("Stopping reactor") 139 self.reactor.stop()

140 d.addCallback(shutdown) 141 return d

142

143 - def cancelCleanShutdown(self):

144 if not self.shuttingDown: 145 return 146 log.msg("Cancelling clean shutdown") 147 self.shuttingDown = False

148

149 - def _sortfunc(self, b1, b2):

150 t1 = b1.getOldestRequestTime() 151 t2 = b2.getOldestRequestTime() 152 # If t1 or t2 is None, then there are no build requests, 153 # so sort it at the end 154 if t1 is None: 155 return 1 156 if t2 is None: 157 return -1 158 return cmp(t1, t2)

159

160 - def _sort_builders(self, parent, builders):

161 return sorted(builders, self._sortfunc)

162

163 - def _get_processors(self):

164 if self.shuttingDown: 165 return [] 166 builders = self.builders.values() 167 sorter = self.prioritizeBuilders or self._sort_builders 168 try: 169 builders = sorter(self.parent, builders) 170 except: 171 log.msg("Exception prioritizing builders") 172 log.err(Failure()) 173 # leave them in the original order 174 return [b.run for b in builders]

175

176 - def trigger_add_buildrequest(self, category, *brids):

177 # a buildrequest has been added or resubmitted 178 self.loop.trigger()

179 - def triggerNewBuildCheck(self):

180 # called when a build finishes, or a slave attaches 181 self.loop.trigger()

182 183 # these four are convenience functions for testing 184

185 - def waitUntilBuilderAttached(self, name):

186 b = self.builders[name] 187 #if b.slaves: 188 # return defer.succeed(None) 189 d = defer.Deferred() 190 b.watchers['attach'].append(d) 191 return d

192

193 - def waitUntilBuilderDetached(self, name):

194 b = self.builders.get(name) 195 if not b or not b.slaves: 196 return defer.succeed(None) 197 d = defer.Deferred() 198 b.watchers['detach'].append(d) 199 return d

200

201 - def waitUntilBuilderFullyDetached(self, name):

202 b = self.builders.get(name) 203 # TODO: this looks too deeply inside the Builder object 204 if not b or not b.slaves: 205 return defer.succeed(None) 206 d = defer.Deferred() 207 b.watchers['detach_all'].append(d) 208 return d

209

210 - def waitUntilBuilderIdle(self, name):

211 b = self.builders[name] 212 # TODO: this looks way too deeply inside the Builder object 213 for sb in b.slaves: 214 if sb.state != IDLE: 215 d = defer.Deferred() 216 b.watchers['idle'].append(d) 217 return d 218 return defer.succeed(None)

219

220 - def loadConfig_Slaves(self, new_slaves):

221 new_portnum = (self.lastSlavePortnum is not None 222 and self.lastSlavePortnum != self.master.slavePortnum) 223 if new_portnum: 224 # it turns out this is pretty hard.. 225 raise ValueError("changing slavePortnum in reconfig is not supported") 226 self.lastSlavePortnum = self.master.slavePortnum 227 228 old_slaves = [c for c in list(self) 229 if interfaces.IBuildSlave.providedBy(c)] 230 231 # identify added/removed slaves. For each slave we construct a tuple 232 # of (name, password, class), and we consider the slave to be already 233 # present if the tuples match. (we include the class to make sure 234 # that BuildSlave(name,pw) is different than 235 # SubclassOfBuildSlave(name,pw) ). If the password or class has 236 # changed, we will remove the old version of the slave and replace it 237 # with a new one. If anything else has changed, we just update the 238 # old BuildSlave instance in place. If the name has changed, of 239 # course, it looks exactly the same as deleting one slave and adding 240 # an unrelated one. 241 old_t = {} 242 for s in old_slaves: 243 old_t[(s.slavename, s.password, s.__class__)] = s 244 new_t = {} 245 for s in new_slaves: 246 new_t[(s.slavename, s.password, s.__class__)] = s 247 removed = [old_t[t] 248 for t in old_t 249 if t not in new_t] 250 added = [new_t[t] 251 for t in new_t 252 if t not in old_t] 253 remaining_t = [t 254 for t in new_t 255 if t in old_t] 256 257 # removeSlave will hang up on the old bot 258 dl = [] 259 for s in removed: 260 dl.append(self.removeSlave(s)) 261 d = defer.DeferredList(dl, fireOnOneErrback=True) 262 263 def add_new(res): 264 for s in added: 265 self.addSlave(s)

266 d.addCallback(add_new) 267 268 def update_remaining(_): 269 for t in remaining_t: 270 old_t[t].update(new_t[t]) 271 d.addCallback(update_remaining) 272 273 return d 274

275 - def addSlave(self, s):

276 s.setServiceParent(self) 277 s.setBotmaster(self) 278 self.slaves[s.slavename] = s 279 s.pb_registration = self.master.pbmanager.register( 280 self.master.slavePortnum, s.slavename, 281 s.password, self.getPerspective)

282

283 - def removeSlave(self, s):

284 d = s.disownServiceParent() 285 d.addCallback(lambda _ : s.pb_registration.unregister()) 286 d.addCallback(lambda _ : self.slaves[s.slavename].disconnect()) 287 def delslave(_): 288 del self.slaves[s.slavename]

289 d.addCallback(delslave) 290 return d 291

292 - def slaveLost(self, bot):

293 for name, b in self.builders.items(): 294 if bot.slavename in b.slavenames: 295 b.detached(bot)

296

297 - def getBuildersForSlave(self, slavename):

298 return [b 299 for b in self.builders.values() 300 if slavename in b.slavenames]

301

302 - def getBuildernames(self):

303 return self.builderNames

304

305 - def getBuilders(self):

306 allBuilders = [self.builders[name] for name in self.builderNames] 307 return allBuilders

308

309 - def setBuilders(self, builders):

310 # TODO: remove self.builders and just use the Service hierarchy to 311 # keep track of active builders. We could keep self.builderNames to 312 # retain ordering, if it seems important. 313 self.builders = {} 314 self.builderNames = [] 315 d = defer.DeferredList([b.disownServiceParent() for b in list(self) 316 if isinstance(b, Builder)], 317 fireOnOneErrback=True) 318 def _add(ign): 319 log.msg("setBuilders._add: %s %s" % (list(self), builders)) 320 for b in builders: 321 for slavename in b.slavenames: 322 # this is actually validated earlier 323 assert slavename in self.slaves 324 self.builders[b.name] = b 325 self.builderNames.append(b.name) 326 b.setBotmaster(self) 327 b.setServiceParent(self)

328 d.addCallback(_add) 329 d.addCallback(lambda ign: self._updateAllSlaves()) 330 return d 331

332 - def _updateAllSlaves(self):

333 """Notify all buildslaves about changes in their Builders.""" 334 dl = [] 335 for s in self.slaves.values(): 336 d = s.updateSlave() 337 d.addErrback(log.err) 338 dl.append(d) 339 return defer.DeferredList(dl)

340

341 - def shouldMergeRequests(self, builder, req1, req2):

342 """Determine whether two BuildRequests should be merged for 343 the given builder. 344 345 """ 346 if self.mergeRequests is not None: 347 if callable(self.mergeRequests): 348 return self.mergeRequests(builder, req1, req2) 349 elif self.mergeRequests == False: 350 # To save typing, this allows c['mergeRequests'] = False 351 return False 352 return req1.canBeMergedWith(req2)

353

354 - def getPerspective(self, mind, slavename):

355 sl = self.slaves[slavename] 356 if not sl: 357 return None 358 359 # record when this connection attempt occurred 360 sl.recordConnectTime() 361 362 if sl.isConnected(): 363 # duplicate slave - send it to arbitration 364 arb = DuplicateSlaveArbitrator(sl) 365 return arb.getPerspective(mind, slavename) 366 else: 367 log.msg("slave '%s' attaching from %s" % (slavename, mind.broker.transport.getPeer())) 368 return sl

369

370 - def stopService(self):

371 for b in self.builders.values(): 372 b.builder_status.addPointEvent(["master", "shutdown"]) 373 b.builder_status.saveYourself() 374 return service.MultiService.stopService(self)

375

376 - def getLockByID(self, lockid):

377 """Convert a Lock identifier into an actual Lock instance. 378 @param lockid: a locks.MasterLock or locks.SlaveLock instance 379 @return: a locks.RealMasterLock or locks.RealSlaveLock instance 380 """ 381 assert isinstance(lockid, (locks.MasterLock, locks.SlaveLock)) 382 if not lockid in self.locks: 383 self.locks[lockid] = lockid.lockClass(lockid) 384 # if the master.cfg file has changed maxCount= on the lock, the next 385 # time a build is started, they'll get a new RealLock instance. Note 386 # that this requires that MasterLock and SlaveLock (marker) instances 387 # be hashable and that they should compare properly. 388 return self.locks[lockid]

389

390 -class DuplicateSlaveArbitrator(object):

391 """Utility class to arbitrate the situation when a new slave connects with 392 the name of an existing, connected slave""" 393 # There are several likely duplicate slave scenarios in practice: 394 # 395 # 1. two slaves are configured with the same username/password 396 # 397 # 2. the same slave process believes it is disconnected (due to a network 398 # hiccup), and is trying to reconnect 399 # 400 # For the first case, we want to prevent the two slaves from repeatedly 401 # superseding one another (which results in lots of failed builds), so we 402 # will prefer the old slave. However, for the second case we need to 403 # detect situations where the old slave is "gone". Sometimes "gone" means 404 # that the TCP/IP connection to it is in a long timeout period (10-20m, 405 # depending on the OS configuration), so this can take a while. 406 407 PING_TIMEOUT = 10 408 """Timeout for pinging the old slave. Set this to something quite long, as 409 a very busy slave (e.g., one sending a big log chunk) may take a while to 410 return a ping.""" 411

412 - def __init__(self, slave):

413 self.old_slave = slave 414 "L{buildbot.buildslave.AbstractSlaveBuilder} instance"

415

416 - def getPerspective(self, mind, slavename):

417 self.new_slave_mind = mind 418 419 old_tport = self.old_slave.slave.broker.transport 420 new_tport = mind.broker.transport 421 log.msg("duplicate slave %s; delaying new slave (%s) and pinging old (%s)" % 422 (self.old_slave.slavename, new_tport.getPeer(), old_tport.getPeer())) 423 424 # delay the new slave until we decide what to do with it 425 self.new_slave_d = defer.Deferred() 426 427 # Ping the old slave. If this kills it, then we can allow the new 428 # slave to connect. If this does not kill it, then we disconnect 429 # the new slave. 430 self.ping_old_slave_done = False 431 self.old_slave_connected = True 432 self.ping_old_slave(new_tport.getPeer()) 433 434 # Print a message on the new slave, if possible. 435 self.ping_new_slave_done = False 436 self.ping_new_slave() 437 438 return self.new_slave_d

439

440 - def ping_new_slave(self):

441 d = self.new_slave_mind.callRemote("print", 442 "master already has a connection named '%s' - checking its liveness" 443 % self.old_slave.slavename) 444 def done(_): 445 # failure or success, doesn't matter 446 self.ping_new_slave_done = True 447 self.maybe_done()

448 d.addBoth(done)

449

450 - def ping_old_slave(self, new_peer):

451 # set a timer on this ping, in case the network is bad. TODO: a timeout 452 # on the ping itself is not quite what we want. If there is other data 453 # flowing over the PB connection, then we should keep waiting. Bug #1703 454 def timeout(): 455 self.ping_old_slave_timeout = None 456 self.ping_old_slave_timed_out = True 457 self.old_slave_connected = False 458 self.ping_old_slave_done = True 459 self.maybe_done()

460 self.ping_old_slave_timeout = reactor.callLater(self.PING_TIMEOUT, timeout) 461 self.ping_old_slave_timed_out = False 462 463 d = self.old_slave.slave.callRemote("print", 464 "master got a duplicate connection from %s; keeping this one" % new_peer) 465 466 def clear_timeout(r): 467 if self.ping_old_slave_timeout: 468 self.ping_old_slave_timeout.cancel() 469 self.ping_old_slave_timeout = None 470 return r 471 d.addBoth(clear_timeout) 472 473 def old_gone(f): 474 if self.ping_old_slave_timed_out: 475 return # ignore after timeout 476 f.trap(pb.PBConnectionLost) 477 log.msg(("connection lost while pinging old slave '%s' - " + 478 "keeping new slave") % self.old_slave.slavename) 479 self.old_slave_connected = False 480 d.addErrback(old_gone) 481 482 def other_err(f): 483 if self.ping_old_slave_timed_out: 484 return # ignore after timeout 485 log.msg("unexpected error while pinging old slave; disconnecting it") 486 log.err(f) 487 self.old_slave_connected = False 488 d.addErrback(other_err) 489 490 def done(_): 491 if self.ping_old_slave_timed_out: 492 return # ignore after timeout 493 self.ping_old_slave_done = True 494 self.maybe_done() 495 d.addCallback(done) 496

497 - def maybe_done(self):

498 if not self.ping_new_slave_done or not self.ping_old_slave_done: 499 return 500 501 # both pings are done, so sort out the results 502 if self.old_slave_connected: 503 self.disconnect_new_slave() 504 else: 505 self.start_new_slave()

506

507 - def start_new_slave(self, count=20):

508 if not self.new_slave_d: 509 return 510 511 # we need to wait until the old slave has actually disconnected, which 512 # can take a little while -- but don't wait forever! 513 if self.old_slave.isConnected(): 514 if self.old_slave.slave: 515 self.old_slave.slave.broker.transport.loseConnection() 516 if count < 0: 517 log.msg("WEIRD: want to start new slave, but the old slave will not disconnect") 518 self.disconnect_new_slave() 519 else: 520 reactor.callLater(0.1, self.start_new_slave, count-1) 521 return 522 523 d = self.new_slave_d 524 self.new_slave_d = None 525 d.callback(self.old_slave)

526

527 - def disconnect_new_slave(self):

528 if not self.new_slave_d: 529 return 530 d = self.new_slave_d 531 self.new_slave_d = None 532 log.msg("rejecting duplicate slave with exception") 533 d.errback(Failure(RuntimeError("rejecting duplicate slave")))

534 535 ######################################## 536 537 538

539 -class DebugPerspective(NewCredPerspective):

540 - def attached(self, mind):

541 return self

542 - def detached(self, mind):

543 pass

544

545 - def perspective_requestBuild(self, buildername, reason, branch, revision, properties={}):

546 from buildbot.sourcestamp import SourceStamp 547 c = interfaces.IControl(self.master) 548 bc = c.getBuilder(buildername) 549 ss = SourceStamp(branch, revision) 550 bpr = Properties() 551 bpr.update(properties, "remote requestBuild") 552 bc.submitBuildRequest(ss, reason, bpr)

553

554 - def perspective_pingBuilder(self, buildername):

555 c = interfaces.IControl(self.master) 556 bc = c.getBuilder(buildername) 557 bc.ping()

558

559 - def perspective_fakeChange(self, file, revision=None, who="fakeUser", 560 branch=None, repository="", 561 project=""):

562 change = Change(who, [file], "some fake comments\n", 563 branch=branch, revision=revision, 564 repository=repository, project=project) 565 c = interfaces.IControl(self.master) 566 c.addChange(change)

567

568 - def perspective_setCurrentState(self, buildername, state):

569 builder = self.botmaster.builders.get(buildername) 570 if not builder: return 571 if state == "offline": 572 builder.statusbag.currentlyOffline() 573 if state == "idle": 574 builder.statusbag.currentlyIdle() 575 if state == "waiting": 576 builder.statusbag.currentlyWaiting(now()+10) 577 if state == "building": 578 builder.statusbag.currentlyBuilding(None)

579 - def perspective_reload(self):

580 print "doing reload of the config file" 581 self.master.loadTheConfigFile()

582 - def perspective_pokeIRC(self):

583 print "saying something on IRC" 584 from buildbot.status import words 585 for s in self.master: 586 if isinstance(s, words.IRC): 587 bot = s.f 588 for channel in bot.channels: 589 print " channel", channel 590 bot.p.msg(channel, "Ow, quit it")

591

592 - def perspective_print(self, msg):

593 print "debug", msg

594 595 ######################################## 596

597 -class _Unset: pass # marker

598

599 -class LogRotation:

600 '''holds log rotation parameters (for WebStatus)'''

601 - def __init__(self):

602 self.rotateLength = 1 * 1000 * 1000 603 self.maxRotatedFiles = 10

604

605 -class BuildMaster(service.MultiService):

606 debug = 0 607 manhole = None 608 debugPassword = None 609 projectName = "(unspecified)" 610 projectURL = None 611 buildbotURL = None 612 change_svc = None 613 properties = Properties() 614

615 - def __init__(self, basedir, configFileName="master.cfg", db_spec=None):

616 service.MultiService.__init__(self) 617 self.setName("buildmaster") 618 self.basedir = basedir 619 self.configFileName = configFileName 620 621 self.pbmanager = buildbot.pbmanager.PBManager() 622 self.pbmanager.setServiceParent(self) 623 "L{buildbot.pbmanager.PBManager} instance managing connections for this master" 624 625 self.slavePortnum = None 626 self.slavePort = None 627 628 self.change_svc = ChangeManager() 629 self.change_svc.setServiceParent(self) 630 631 try: 632 hostname = os.uname()[1] # only on unix 633 except AttributeError: 634 hostname = "?" 635 self.master_name = "%s:%s" % (hostname, os.path.abspath(self.basedir)) 636 self.master_incarnation = "pid%d-boot%d" % (os.getpid(), time.time()) 637 638 self.botmaster = BotMaster(self) 639 self.botmaster.setName("botmaster") 640 self.botmaster.setMasterName(self.master_name, self.master_incarnation) 641 self.botmaster.setServiceParent(self) 642 643 self.debugClientRegistration = None 644 645 self.status = Status(self.botmaster, self.basedir) 646 self.statusTargets = [] 647 648 self.db = None 649 self.db_url = None 650 self.db_poll_interval = _Unset 651 if db_spec: 652 self.loadDatabase(db_spec) 653 654 # note that "read" here is taken in the past participal (i.e., "I read 655 # the config already") rather than the imperative ("you should read the 656 # config later") 657 self.readConfig = False 658 659 # create log_rotation object and set default parameters (used by WebStatus) 660 self.log_rotation = LogRotation()

661

662 - def startService(self):

663 service.MultiService.startService(self) 664 if not self.readConfig: 665 # TODO: consider catching exceptions during this call to 666 # loadTheConfigFile and bailing (reactor.stop) if it fails, 667 # since without a config file we can't do anything except reload 668 # the config file, and it would be nice for the user to discover 669 # this quickly. 670 self.loadTheConfigFile() 671 if hasattr(signal, "SIGHUP"): 672 signal.signal(signal.SIGHUP, self._handleSIGHUP) 673 for b in self.botmaster.builders.values(): 674 b.builder_status.addPointEvent(["master", "started"]) 675 b.builder_status.saveYourself()

676

677 - def _handleSIGHUP(self, *args):

678 reactor.callLater(0, self.loadTheConfigFile)

679

680 - def getStatus(self):

681 """ 682 @rtype: L{buildbot.status.builder.Status} 683 """ 684 return self.status

685

686 - def loadTheConfigFile(self, configFile=None):

687 if not configFile: 688 configFile = os.path.join(self.basedir, self.configFileName) 689 690 log.msg("Creating BuildMaster -- buildbot.version: %s" % buildbot.version) 691 log.msg("loading configuration from %s" % configFile) 692 configFile = os.path.expanduser(configFile) 693 694 try: 695 f = open(configFile, "r") 696 except IOError, e: 697 log.msg("unable to open config file '%s'" % configFile) 698 log.msg("leaving old configuration in place") 699 log.err(e) 700 return 701 702 try: 703 d = self.loadConfig(f) 704 except: 705 log.msg("error during loadConfig") 706 log.err() 707 log.msg("The new config file is unusable, so I'll ignore it.") 708 log.msg("I will keep using the previous config file instead.") 709 return # sorry unit tests 710 f.close() 711 return d # for unit tests

712

713 - def loadConfig(self, f, check_synchronously_only=False):

714 """Internal function to load a specific configuration file. Any 715 errors in the file will be signalled by raising an exception. 716 717 If check_synchronously_only=True, I will return (with None) 718 synchronously, after checking the config file for sanity, or raise an 719 exception. I may also emit some DeprecationWarnings. 720 721 If check_synchronously_only=False, I will return a Deferred that 722 fires (with None) when the configuration changes have been completed. 723 This may involve a round-trip to each buildslave that was involved.""" 724 725 localDict = {'basedir': os.path.expanduser(self.basedir)} 726 try: 727 exec f in localDict 728 except: 729 log.msg("error while parsing config file") 730 raise 731 732 try: 733 config = localDict['BuildmasterConfig'] 734 except KeyError: 735 log.err("missing config dictionary") 736 log.err("config file must define BuildmasterConfig") 737 raise 738 739 known_keys = ("slaves", "change_source", 740 "schedulers", "builders", "mergeRequests", 741 "slavePortnum", "debugPassword", "logCompressionLimit", 742 "manhole", "status", "projectName", "projectURL", 743 "buildbotURL", "properties", "prioritizeBuilders", 744 "eventHorizon", "buildCacheSize", "changeCacheSize", 745 "logHorizon", "buildHorizon", "changeHorizon", 746 "logMaxSize", "logMaxTailSize", "logCompressionMethod", 747 "db_url", "multiMaster", "db_poll_interval", 748 ) 749 for k in config.keys(): 750 if k not in known_keys: 751 log.msg("unknown key '%s' defined in config dictionary" % k) 752 753 try: 754 # required 755 schedulers = config['schedulers'] 756 builders = config['builders'] 757 slavePortnum = config['slavePortnum'] 758 #slaves = config['slaves'] 759 #change_source = config['change_source'] 760 761 # optional 762 db_url = config.get("db_url", "sqlite:///state.sqlite") 763 db_poll_interval = config.get("db_poll_interval", None) 764 debugPassword = config.get('debugPassword') 765 manhole = config.get('manhole') 766 status = config.get('status', []) 767 projectName = config.get('projectName') 768 projectURL = config.get('projectURL') 769 buildbotURL = config.get('buildbotURL') 770 properties = config.get('properties', {}) 771 buildCacheSize = config.get('buildCacheSize', None) 772 changeCacheSize = config.get('changeCacheSize', None) 773 eventHorizon = config.get('eventHorizon', 50) 774 logHorizon = config.get('logHorizon', None) 775 buildHorizon = config.get('buildHorizon', None) 776 logCompressionLimit = config.get('logCompressionLimit', 4*1024) 777 if logCompressionLimit is not None and not \ 778 isinstance(logCompressionLimit, int): 779 raise ValueError("logCompressionLimit needs to be bool or int") 780 logCompressionMethod = config.get('logCompressionMethod', "bz2") 781 if logCompressionMethod not in ('bz2', 'gz'): 782 raise ValueError("logCompressionMethod needs to be 'bz2', or 'gz'") 783 logMaxSize = config.get('logMaxSize') 784 if logMaxSize is not None and not \ 785 isinstance(logMaxSize, int): 786 raise ValueError("logMaxSize needs to be None or int") 787 logMaxTailSize = config.get('logMaxTailSize') 788 if logMaxTailSize is not None and not \ 789 isinstance(logMaxTailSize, int): 790 raise ValueError("logMaxTailSize needs to be None or int") 791 mergeRequests = config.get('mergeRequests') 792 if mergeRequests not in (None, False) and not callable(mergeRequests): 793 raise ValueError("mergeRequests must be a callable or False") 794 prioritizeBuilders = config.get('prioritizeBuilders') 795 if prioritizeBuilders is not None and not callable(prioritizeBuilders): 796 raise ValueError("prioritizeBuilders must be callable") 797 changeHorizon = config.get("changeHorizon") 798 if changeHorizon is not None and not isinstance(changeHorizon, int): 799 raise ValueError("changeHorizon needs to be an int") 800 801 multiMaster = config.get("multiMaster", False) 802 803 except KeyError: 804 log.msg("config dictionary is missing a required parameter") 805 log.msg("leaving old configuration in place") 806 raise 807 808 if "sources" in config: 809 m = ("c['sources'] is deprecated as of 0.7.6 and is no longer " 810 "accepted in >= 0.8.0 . Please use c['change_source'] instead.") 811 raise KeyError(m) 812 813 if "bots" in config: 814 m = ("c['bots'] is deprecated as of 0.7.6 and is no longer " 815 "accepted in >= 0.8.0 . Please use c['slaves'] instead.") 816 raise KeyError(m) 817 818 slaves = config.get('slaves', []) 819 if "slaves" not in config: 820 log.msg("config dictionary must have a 'slaves' key") 821 log.msg("leaving old configuration in place") 822 raise KeyError("must have a 'slaves' key") 823 824 if changeHorizon is not None: 825 self.change_svc.changeHorizon = changeHorizon 826 827 change_source = config.get('change_source', []) 828 if isinstance(change_source, (list, tuple)): 829 change_sources = change_source 830 else: 831 change_sources = [change_source] 832 833 # do some validation first 834 for s in slaves: 835 assert interfaces.IBuildSlave.providedBy(s) 836 if s.slavename in ("debug", "change", "status"): 837 raise KeyError( 838 "reserved name '%s' used for a bot" % s.slavename) 839 if config.has_key('interlocks'): 840 raise KeyError("c['interlocks'] is no longer accepted") 841 assert self.db_url is None or db_url == self.db_url, \ 842 "Cannot change db_url after master has started" 843 assert db_poll_interval is None or isinstance(db_poll_interval, int), \ 844 "db_poll_interval must be an integer: seconds between polls" 845 assert self.db_poll_interval is _Unset or db_poll_interval == self.db_poll_interval, \ 846 "Cannot change db_poll_interval after master has started" 847 848 assert isinstance(change_sources, (list, tuple)) 849 for s in change_sources: 850 assert interfaces.IChangeSource(s, None) 851 # this assertion catches c['schedulers'] = Scheduler(), since 852 # Schedulers are service.MultiServices and thus iterable. 853 errmsg = "c['schedulers'] must be a list of Scheduler instances" 854 assert isinstance(schedulers, (list, tuple)), errmsg 855 for s in schedulers: 856 assert interfaces.IScheduler(s, None), errmsg 857 assert isinstance(status, (list, tuple)) 858 for s in status: 859 assert interfaces.IStatusReceiver(s, None) 860 861 slavenames = [s.slavename for s in slaves] 862 buildernames = [] 863 dirnames = [] 864 865 # convert builders from objects to config dictionaries 866 builders_dicts = [] 867 for b in builders: 868 if isinstance(b, BuilderConfig): 869 builders_dicts.append(b.getConfigDict()) 870 elif type(b) is dict: 871 builders_dicts.append(b) 872 else: 873 raise ValueError("builder %s is not a BuilderConfig object (or a dict)" % b) 874 builders = builders_dicts 875 876 for b in builders: 877 if b.has_key('slavename') and b['slavename'] not in slavenames: 878 raise ValueError("builder %s uses undefined slave %s" \ 879 % (b['name'], b['slavename'])) 880 for n in b.get('slavenames', []): 881 if n not in slavenames: 882 raise ValueError("builder %s uses undefined slave %s" \ 883 % (b['name'], n)) 884 if b['name'] in buildernames: 885 raise ValueError("duplicate builder name %s" 886 % b['name']) 887 buildernames.append(b['name']) 888 889 # sanity check name (BuilderConfig does this too) 890 if b['name'].startswith("_"): 891 errmsg = ("builder names must not start with an " 892 "underscore: " + b['name']) 893 log.err(errmsg) 894 raise ValueError(errmsg) 895 896 # Fix the dictionary with default values, in case this wasn't 897 # specified with a BuilderConfig object (which sets the same defaults) 898 b.setdefault('builddir', safeTranslate(b['name'])) 899 b.setdefault('slavebuilddir', b['builddir']) 900 b.setdefault('buildHorizon', buildHorizon) 901 b.setdefault('logHorizon', logHorizon) 902 b.setdefault('eventHorizon', eventHorizon) 903 if b['builddir'] in dirnames: 904 raise ValueError("builder %s reuses builddir %s" 905 % (b['name'], b['builddir'])) 906 dirnames.append(b['builddir']) 907 908 unscheduled_buildernames = buildernames[:] 909 schedulernames = [] 910 for s in schedulers: 911 for b in s.listBuilderNames(): 912 # Skip checks for builders in multimaster mode 913 if not multiMaster: 914 assert b in buildernames, \ 915 "%s uses unknown builder %s" % (s, b) 916 if b in unscheduled_buildernames: 917 unscheduled_buildernames.remove(b) 918 919 if s.name in schedulernames: 920 msg = ("Schedulers must have unique names, but " 921 "'%s' was a duplicate" % (s.name,)) 922 raise ValueError(msg) 923 schedulernames.append(s.name) 924 925 # Skip the checks for builders in multimaster mode 926 if not multiMaster and unscheduled_buildernames: 927 log.msg("Warning: some Builders have no Schedulers to drive them:" 928 " %s" % (unscheduled_buildernames,)) 929 930 # assert that all locks used by the Builds and their Steps are 931 # uniquely named. 932 lock_dict = {} 933 for b in builders: 934 for l in b.get('locks', []): 935 if isinstance(l, locks.LockAccess): # User specified access to the lock 936 l = l.lockid 937 if lock_dict.has_key(l.name): 938 if lock_dict[l.name] is not l: 939 raise ValueError("Two different locks (%s and %s) " 940 "share the name %s" 941 % (l, lock_dict[l.name], l.name)) 942 else: 943 lock_dict[l.name] = l 944 # TODO: this will break with any BuildFactory that doesn't use a 945 # .steps list, but I think the verification step is more 946 # important. 947 for s in b['factory'].steps: 948 for l in s[1].get('locks', []): 949 if isinstance(l, locks.LockAccess): # User specified access to the lock 950 l = l.lockid 951 if lock_dict.has_key(l.name): 952 if lock_dict[l.name] is not l: 953 raise ValueError("Two different locks (%s and %s)" 954 " share the name %s" 955 % (l, lock_dict[l.name], l.name)) 956 else: 957 lock_dict[l.name] = l 958 959 if not isinstance(properties, dict): 960 raise ValueError("c['properties'] must be a dictionary") 961 962 # slavePortnum supposed to be a strports specification 963 if type(slavePortnum) is int: 964 slavePortnum = "tcp:%d" % slavePortnum 965 966 if check_synchronously_only: 967 return 968 # now we're committed to implementing the new configuration, so do 969 # it atomically 970 # TODO: actually, this is spread across a couple of Deferreds, so it 971 # really isn't atomic. 972 973 d = defer.succeed(None) 974 975 self.projectName = projectName 976 self.projectURL = projectURL 977 self.buildbotURL = buildbotURL 978 979 self.properties = Properties() 980 self.properties.update(properties, self.configFileName) 981 982 self.status.logCompressionLimit = logCompressionLimit 983 self.status.logCompressionMethod = logCompressionMethod 984 self.status.logMaxSize = logMaxSize 985 self.status.logMaxTailSize = logMaxTailSize 986 # Update any of our existing builders with the current log parameters. 987 # This is required so that the new value is picked up after a 988 # reconfig. 989 for builder in self.botmaster.builders.values(): 990 builder.builder_status.setLogCompressionLimit(logCompressionLimit) 991 builder.builder_status.setLogCompressionMethod(logCompressionMethod) 992 builder.builder_status.setLogMaxSize(logMaxSize) 993 builder.builder_status.setLogMaxTailSize(logMaxTailSize) 994 995 if mergeRequests is not None: 996 self.botmaster.mergeRequests = mergeRequests 997 if prioritizeBuilders is not None: 998 self.botmaster.prioritizeBuilders = prioritizeBuilders 999 1000 self.buildCacheSize = buildCacheSize 1001 self.changeCacheSize = changeCacheSize 1002 self.eventHorizon = eventHorizon 1003 self.logHorizon = logHorizon 1004 self.buildHorizon = buildHorizon 1005 self.slavePortnum = slavePortnum # TODO: move this to master.config.slavePortnum 1006 1007 # Set up the database 1008 d.addCallback(lambda res: 1009 self.loadConfig_Database(db_url, db_poll_interval)) 1010 1011 # set up slaves 1012 d.addCallback(lambda res: self.loadConfig_Slaves(slaves)) 1013 1014 # self.manhole 1015 if manhole != self.manhole: 1016 # changing 1017 if self.manhole: 1018 # disownServiceParent may return a Deferred 1019 d.addCallback(lambda res: self.manhole.disownServiceParent()) 1020 def _remove(res): 1021 self.manhole = None 1022 return res

1023 d.addCallback(_remove) 1024 if manhole: 1025 def _add(res): 1026 self.manhole = manhole 1027 manhole.setServiceParent(self)

1028 d.addCallback(_add) 1029 1030 # add/remove self.botmaster.builders to match builders. The 1031 # botmaster will handle startup/shutdown issues. 1032 d.addCallback(lambda res: self.loadConfig_Builders(builders)) 1033 1034 d.addCallback(lambda res: self.loadConfig_status(status)) 1035 1036 # Schedulers are added after Builders in case they start right away 1037 d.addCallback(lambda res: 1038 self.scheduler_manager.updateSchedulers(schedulers)) 1039 1040 # and Sources go after Schedulers for the same reason 1041 d.addCallback(lambda res: self.loadConfig_Sources(change_sources)) 1042 1043 # debug client 1044 d.addCallback(lambda res: self.loadConfig_DebugClient(debugPassword)) 1045 1046 log.msg("configuration update started") 1047 def _done(res): 1048 self.readConfig = True 1049 log.msg("configuration update complete") 1050 d.addCallback(_done) 1051 d.addCallback(lambda res: self.botmaster.triggerNewBuildCheck()) 1052 d.addErrback(log.err) 1053 return d 1054

1055 - def loadDatabase(self, db_spec, db_poll_interval=None):

1056 if self.db: 1057 return 1058 1059 # make sure it's up to date 1060 sm = DBSchemaManager(db_spec, self.basedir) 1061 if not sm.is_current(): 1062 raise exceptions.DatabaseNotReadyError, textwrap.dedent(""" 1063 The Buildmaster database needs to be upgraded before this version of buildbot 1064 can run. Use the following command-line 1065 buildbot upgrade-master path/to/master 1066 to upgrade the database, and try starting the buildmaster again. You may want 1067 to make a backup of your buildmaster before doing so. If you are using MySQL, 1068 you must specify the connector string on the upgrade-master command line: 1069 buildbot upgrade-master --db=<db-connector-string> path/to/master 1070 """) 1071 1072 self.db = connector.DBConnector(db_spec) 1073 if self.changeCacheSize: 1074 self.db.setChangeCacheSize(self.changeCacheSize) 1075 self.db.start() 1076 1077 self.botmaster.db = self.db 1078 self.status.setDB(self.db) 1079 1080 self.db.subscribe_to("add-buildrequest", 1081 self.botmaster.trigger_add_buildrequest) 1082 1083 sm = SchedulerManager(self, self.db, self.change_svc) 1084 self.db.subscribe_to("add-change", sm.trigger_add_change) 1085 self.db.subscribe_to("modify-buildset", sm.trigger_modify_buildset) 1086 1087 self.scheduler_manager = sm 1088 sm.setServiceParent(self) 1089 1090 # Set db_poll_interval (perhaps to 30 seconds) if you are using 1091 # multiple buildmasters that share a common database, such that the 1092 # masters need to discover what each other is doing by polling the 1093 # database. TODO: this will be replaced by the DBNotificationServer. 1094 if db_poll_interval: 1095 # it'd be nice if TimerService let us set now=False 1096 t1 = TimerService(db_poll_interval, sm.trigger) 1097 t1.setServiceParent(self) 1098 t2 = TimerService(db_poll_interval, self.botmaster.loop.trigger) 1099 t2.setServiceParent(self)

1100 # adding schedulers (like when loadConfig happens) will trigger the 1101 # scheduler loop at least once, which we need to jump-start things 1102 # like Periodic. 1103

1104 - def loadConfig_Database(self, db_url, db_poll_interval):

1105 self.db_url = db_url 1106 self.db_poll_interval = db_poll_interval 1107 db_spec = DBSpec.from_url(db_url, self.basedir) 1108 self.loadDatabase(db_spec, db_poll_interval)

1109

1110 - def loadConfig_Slaves(self, new_slaves):

1111 return self.botmaster.loadConfig_Slaves(new_slaves)

1112

1113 - def loadConfig_Sources(self, sources):

1114 if not sources: 1115 log.msg("warning: no ChangeSources specified in c['change_source']") 1116 # shut down any that were removed, start any that were added 1117 deleted_sources = [s for s in self.change_svc if s not in sources] 1118 added_sources = [s for s in sources if s not in self.change_svc] 1119 log.msg("adding %d new changesources, removing %d" % 1120 (len(added_sources), len(deleted_sources))) 1121 dl = [self.change_svc.removeSource(s) for s in deleted_sources] 1122 def addNewOnes(res): 1123 [self.change_svc.addSource(s) for s in added_sources]

1124 d = defer.DeferredList(dl, fireOnOneErrback=1, consumeErrors=0) 1125 d.addCallback(addNewOnes) 1126 return d 1127

1128 - def loadConfig_DebugClient(self, debugPassword):

1129 def makeDbgPerspective(): 1130 persp = DebugPerspective() 1131 persp.master = self 1132 persp.botmaster = self.botmaster 1133 return persp

1134 1135 # unregister the old name.. 1136 if self.debugClientRegistration: 1137 d = self.debugClientRegistration.unregister() 1138 self.debugClientRegistration = None 1139 else: 1140 d = defer.succeed(None) 1141 1142 # and register the new one 1143 def reg(_): 1144 if debugPassword: 1145 self.debugClientRegistration = self.pbmanager.register( 1146 self.slavePortnum, "debug", debugPassword, makeDbgPerspective) 1147 d.addCallback(reg) 1148 return d 1149

1150 - def allSchedulers(self):

1151 return list(self.scheduler_manager)

1152

1153 - def loadConfig_Builders(self, newBuilderData):

1154 somethingChanged = False 1155 newList = {} 1156 newBuilderNames = [] 1157 allBuilders = self.botmaster.builders.copy() 1158 for data in newBuilderData: 1159 name = data['name'] 1160 newList[name] = data 1161 newBuilderNames.append(name) 1162 1163 # identify all that were removed 1164 for oldname in self.botmaster.getBuildernames(): 1165 if oldname not in newList: 1166 log.msg("removing old builder %s" % oldname) 1167 del allBuilders[oldname] 1168 somethingChanged = True 1169 # announce the change 1170 self.status.builderRemoved(oldname) 1171 1172 # everything in newList is either unchanged, changed, or new 1173 for name, data in newList.items(): 1174 old = self.botmaster.builders.get(name) 1175 basedir = data['builddir'] 1176 #name, slave, builddir, factory = data 1177 if not old: # new 1178 # category added after 0.6.2 1179 category = data.get('category', None) 1180 log.msg("adding new builder %s for category %s" % 1181 (name, category)) 1182 statusbag = self.status.builderAdded(name, basedir, category) 1183 builder = Builder(data, statusbag) 1184 allBuilders[name] = builder 1185 somethingChanged = True 1186 elif old.compareToSetup(data): 1187 # changed: try to minimize the disruption and only modify the 1188 # pieces that really changed 1189 diffs = old.compareToSetup(data) 1190 log.msg("updating builder %s: %s" % (name, "\n".join(diffs))) 1191 1192 statusbag = old.builder_status 1193 statusbag.saveYourself() # seems like a good idea 1194 # TODO: if the basedir was changed, we probably need to make 1195 # a new statusbag 1196 new_builder = Builder(data, statusbag) 1197 new_builder.consumeTheSoulOfYourPredecessor(old) 1198 # that migrates any retained slavebuilders too 1199 1200 # point out that the builder was updated. On the Waterfall, 1201 # this will appear just after any currently-running builds. 1202 statusbag.addPointEvent(["config", "updated"]) 1203 1204 allBuilders[name] = new_builder 1205 somethingChanged = True 1206 else: 1207 # unchanged: leave it alone 1208 log.msg("builder %s is unchanged" % name) 1209 pass 1210 1211 # regardless of whether anything changed, get each builder status 1212 # to update its config 1213 for builder in allBuilders.values(): 1214 builder.builder_status.reconfigFromBuildmaster(self) 1215 1216 # and then tell the botmaster if anything's changed 1217 if somethingChanged: 1218 sortedAllBuilders = [allBuilders[name] for name in newBuilderNames] 1219 d = self.botmaster.setBuilders(sortedAllBuilders) 1220 return d 1221 return None

1222

1223 - def loadConfig_status(self, status):

1224 dl = [] 1225 1226 # remove old ones 1227 for s in self.statusTargets[:]: 1228 if not s in status: 1229 log.msg("removing IStatusReceiver", s) 1230 d = defer.maybeDeferred(s.disownServiceParent) 1231 dl.append(d) 1232 self.statusTargets.remove(s) 1233 # after those are finished going away, add new ones 1234 def addNewOnes(res): 1235 for s in status: 1236 if not s in self.statusTargets: 1237 log.msg("adding IStatusReceiver", s) 1238 s.setServiceParent(self) 1239 self.statusTargets.append(s)

1240 d = defer.DeferredList(dl, fireOnOneErrback=1) 1241 d.addCallback(addNewOnes) 1242 return d 1243 1244

1245 - def addChange(self, change):

1246 self.db.addChangeToDatabase(change) 1247 self.status.changeAdded(change)

1248

1249 - def triggerSlaveManager(self):

1250 self.botmaster.triggerNewBuildCheck()

1251

1252 - def submitBuildSet(self, builderNames, ss, reason, props=None, now=False):

1253 # determine the set of Builders to use 1254 for name in builderNames: 1255 b = self.botmaster.builders.get(name) 1256 if not b: 1257 raise KeyError("no such builder named '%s'" % name) 1258 if now and not b.slaves: 1259 raise interfaces.NoSlaveError 1260 if props is None: 1261 props = Properties() 1262 bsid = self.db.runInteractionNow(self._txn_submitBuildSet, 1263 builderNames, ss, reason, props) 1264 return BuildSetStatus(bsid, self.status, self.db)

1265

1266 - def _txn_submitBuildSet(self, t, builderNames, ss, reason, props):

1267 ssid = self.db.get_sourcestampid(ss, t) 1268 bsid = self.db.create_buildset(ssid, reason, props, builderNames, t) 1269 return bsid

1270 1271

1272 -class Control:

1273 implements(interfaces.IControl) 1274

1275 - def __init__(self, master):

1276 self.master = master

1277

1278 - def addChange(self, change):

1279 self.master.addChange(change)

1280

1281 - def submitBuildSet(self, builderNames, ss, reason, props=None, now=False):

1282 return self.master.submitBuildSet(builderNames, ss, reason, props, now)

1283

1284 - def getBuilder(self, name):

1285 b = self.master.botmaster.builders[name] 1286 return BuilderControl(b, self)

1287 1288 components.registerAdapter(Control, BuildMaster, interfaces.IControl) 1289

Source Code for Module buildbot.master