buildbot.master

1 # -*- test-case-name: buildbot.test.test_run -*- 2 3 import os 4 signal = None 5 try: 6 import signal 7 except ImportError: 8 pass 9 from cPickle import load 10 import warnings 11 12 from zope.interface import implements 13 from twisted.python import log, components 14 from twisted.python.failure import Failure 15 from twisted.internet import defer, reactor 16 from twisted.spread import pb 17 from twisted.cred import portal, checkers 18 from twisted.application import service, strports 19 from twisted.persisted import styles 20 21 import buildbot 22 # sibling imports 23 from buildbot.util import now, safeTranslate 24 from buildbot.pbutil import NewCredPerspective 25 from buildbot.process.builder import Builder, IDLE 26 from buildbot.process.base import BuildRequest 27 from buildbot.status.builder import Status 28 from buildbot.changes.changes import Change, ChangeMaster, TestChangeMaster 29 from buildbot.sourcestamp import SourceStamp 30 from buildbot.buildslave import BuildSlave 31 from buildbot import interfaces, locks 32 from buildbot.process.properties import Properties 33 from buildbot.config import BuilderConfig 34 35 ######################################## 36

37 -class BotMaster(service.MultiService):

38 39 """This is the master-side service which manages remote buildbot slaves. 40 It provides them with BuildSlaves, and distributes file change 41 notification messages to them. 42 """ 43 44 debug = 0 45

46 - def __init__(self):

47 service.MultiService.__init__(self) 48 self.builders = {} 49 self.builderNames = [] 50 # builders maps Builder names to instances of bb.p.builder.Builder, 51 # which is the master-side object that defines and controls a build. 52 # They are added by calling botmaster.addBuilder() from the startup 53 # code. 54 55 # self.slaves contains a ready BuildSlave instance for each 56 # potential buildslave, i.e. all the ones listed in the config file. 57 # If the slave is connected, self.slaves[slavename].slave will 58 # contain a RemoteReference to their Bot instance. If it is not 59 # connected, that attribute will hold None. 60 self.slaves = {} # maps slavename to BuildSlave 61 self.statusClientService = None 62 self.watchers = {} 63 64 # self.locks holds the real Lock instances 65 self.locks = {} 66 67 # self.mergeRequests is the callable override for merging build 68 # requests 69 self.mergeRequests = None 70 71 # self.prioritizeBuilders is the callable override for builder order 72 # traversal 73 self.prioritizeBuilders = None

74 75 # these four are convenience functions for testing 76

77 - def waitUntilBuilderAttached(self, name):

78 b = self.builders[name] 79 #if b.slaves: 80 # return defer.succeed(None) 81 d = defer.Deferred() 82 b.watchers['attach'].append(d) 83 return d

84

85 - def waitUntilBuilderDetached(self, name):

86 b = self.builders.get(name) 87 if not b or not b.slaves: 88 return defer.succeed(None) 89 d = defer.Deferred() 90 b.watchers['detach'].append(d) 91 return d

92

93 - def waitUntilBuilderFullyDetached(self, name):

94 b = self.builders.get(name) 95 # TODO: this looks too deeply inside the Builder object 96 if not b or not b.slaves: 97 return defer.succeed(None) 98 d = defer.Deferred() 99 b.watchers['detach_all'].append(d) 100 return d

101

102 - def waitUntilBuilderIdle(self, name):

103 b = self.builders[name] 104 # TODO: this looks way too deeply inside the Builder object 105 for sb in b.slaves: 106 if sb.state != IDLE: 107 d = defer.Deferred() 108 b.watchers['idle'].append(d) 109 return d 110 return defer.succeed(None)

111

112 - def loadConfig_Slaves(self, new_slaves):

113 old_slaves = [c for c in list(self) 114 if interfaces.IBuildSlave.providedBy(c)] 115 116 # identify added/removed slaves. For each slave we construct a tuple 117 # of (name, password, class), and we consider the slave to be already 118 # present if the tuples match. (we include the class to make sure 119 # that BuildSlave(name,pw) is different than 120 # SubclassOfBuildSlave(name,pw) ). If the password or class has 121 # changed, we will remove the old version of the slave and replace it 122 # with a new one. If anything else has changed, we just update the 123 # old BuildSlave instance in place. If the name has changed, of 124 # course, it looks exactly the same as deleting one slave and adding 125 # an unrelated one. 126 old_t = {} 127 for s in old_slaves: 128 old_t[(s.slavename, s.password, s.__class__)] = s 129 new_t = {} 130 for s in new_slaves: 131 new_t[(s.slavename, s.password, s.__class__)] = s 132 removed = [old_t[t] 133 for t in old_t 134 if t not in new_t] 135 added = [new_t[t] 136 for t in new_t 137 if t not in old_t] 138 remaining_t = [t 139 for t in new_t 140 if t in old_t] 141 # removeSlave will hang up on the old bot 142 dl = [] 143 for s in removed: 144 dl.append(self.removeSlave(s)) 145 d = defer.DeferredList(dl, fireOnOneErrback=True) 146 def _add(res): 147 for s in added: 148 self.addSlave(s) 149 for t in remaining_t: 150 old_t[t].update(new_t[t])

151 d.addCallback(_add) 152 return d

153

154 - def addSlave(self, s):

155 s.setServiceParent(self) 156 s.setBotmaster(self) 157 self.slaves[s.slavename] = s

158

159 - def removeSlave(self, s):

160 # TODO: technically, disownServiceParent could return a Deferred 161 s.disownServiceParent() 162 d = self.slaves[s.slavename].disconnect() 163 del self.slaves[s.slavename] 164 return d

165

166 - def slaveLost(self, bot):

167 for name, b in self.builders.items(): 168 if bot.slavename in b.slavenames: 169 b.detached(bot)

170

171 - def getBuildersForSlave(self, slavename):

172 return [b 173 for b in self.builders.values() 174 if slavename in b.slavenames]

175

176 - def getBuildernames(self):

177 return self.builderNames

178

179 - def getBuilders(self):

180 allBuilders = [self.builders[name] for name in self.builderNames] 181 return allBuilders

182

183 - def setBuilders(self, builders):

184 self.builders = {} 185 self.builderNames = [] 186 for b in builders: 187 for slavename in b.slavenames: 188 # this is actually validated earlier 189 assert slavename in self.slaves 190 self.builders[b.name] = b 191 self.builderNames.append(b.name) 192 b.setBotmaster(self) 193 d = self._updateAllSlaves() 194 return d

195

196 - def _updateAllSlaves(self):

197 """Notify all buildslaves about changes in their Builders.""" 198 dl = [s.updateSlave() for s in self.slaves.values()] 199 return defer.DeferredList(dl)

200

201 - def maybeStartAllBuilds(self):

202 builders = self.builders.values() 203 if self.prioritizeBuilders is not None: 204 try: 205 builders = self.prioritizeBuilders(self.parent, builders) 206 except: 207 log.msg("Exception prioritizing builders") 208 log.err(Failure()) 209 return 210 else: 211 def _sortfunc(b1, b2): 212 t1 = b1.getOldestRequestTime() 213 t2 = b2.getOldestRequestTime() 214 # If t1 or t2 is None, then there are no build requests, 215 # so sort it at the end 216 if t1 is None: 217 return 1 218 if t2 is None: 219 return -1 220 return cmp(t1, t2)

221 builders.sort(_sortfunc) 222 try: 223 for b in builders: 224 b.maybeStartBuild() 225 except: 226 log.msg("Exception starting builds") 227 log.err(Failure()) 228

229 - def shouldMergeRequests(self, builder, req1, req2):

230 """Determine whether two BuildRequests should be merged for 231 the given builder. 232 233 """ 234 if self.mergeRequests is not None: 235 return self.mergeRequests(builder, req1, req2) 236 return req1.canBeMergedWith(req2)

237

238 - def getPerspective(self, slavename):

239 return self.slaves[slavename]

240

241 - def shutdownSlaves(self):

242 # TODO: make this into a bot method rather than a builder method 243 for b in self.slaves.values(): 244 b.shutdownSlave()

245

246 - def stopService(self):

247 for b in self.builders.values(): 248 b.builder_status.addPointEvent(["master", "shutdown"]) 249 b.builder_status.saveYourself() 250 return service.Service.stopService(self)

251

252 - def getLockByID(self, lockid):

253 """Convert a Lock identifier into an actual Lock instance. 254 @param lockid: a locks.MasterLock or locks.SlaveLock instance 255 @return: a locks.RealMasterLock or locks.RealSlaveLock instance 256 """ 257 assert isinstance(lockid, (locks.MasterLock, locks.SlaveLock)) 258 if not lockid in self.locks: 259 self.locks[lockid] = lockid.lockClass(lockid) 260 # if the master.cfg file has changed maxCount= on the lock, the next 261 # time a build is started, they'll get a new RealLock instance. Note 262 # that this requires that MasterLock and SlaveLock (marker) instances 263 # be hashable and that they should compare properly. 264 return self.locks[lockid]

265 266 ######################################## 267 268 269

270 -class DebugPerspective(NewCredPerspective):

271 - def attached(self, mind):

272 return self

273 - def detached(self, mind):

274 pass

275

276 - def perspective_requestBuild(self, buildername, reason, branch, revision, properties={}):

277 c = interfaces.IControl(self.master) 278 bc = c.getBuilder(buildername) 279 ss = SourceStamp(branch, revision) 280 bpr = Properties() 281 bpr.update(properties, "remote requestBuild") 282 br = BuildRequest(reason, ss, builderName=buildername, properties=bpr) 283 bc.requestBuild(br)

284

285 - def perspective_pingBuilder(self, buildername):

286 c = interfaces.IControl(self.master) 287 bc = c.getBuilder(buildername) 288 bc.ping()

289

290 - def perspective_fakeChange(self, file, revision=None, who="fakeUser", 291 branch=None):

292 change = Change(who, [file], "some fake comments\n", 293 branch=branch, revision=revision) 294 c = interfaces.IControl(self.master) 295 c.addChange(change)

296

297 - def perspective_setCurrentState(self, buildername, state):

298 builder = self.botmaster.builders.get(buildername) 299 if not builder: return 300 if state == "offline": 301 builder.statusbag.currentlyOffline() 302 if state == "idle": 303 builder.statusbag.currentlyIdle() 304 if state == "waiting": 305 builder.statusbag.currentlyWaiting(now()+10) 306 if state == "building": 307 builder.statusbag.currentlyBuilding(None)

308 - def perspective_reload(self):

309 print "doing reload of the config file" 310 self.master.loadTheConfigFile()

311 - def perspective_pokeIRC(self):

312 print "saying something on IRC" 313 from buildbot.status import words 314 for s in self.master: 315 if isinstance(s, words.IRC): 316 bot = s.f 317 for channel in bot.channels: 318 print " channel", channel 319 bot.p.msg(channel, "Ow, quit it")

320

321 - def perspective_print(self, msg):

322 print "debug", msg

323

324 -class Dispatcher:

325 implements(portal.IRealm) 326

327 - def __init__(self):

328 self.names = {}

329

330 - def register(self, name, afactory):

331 self.names[name] = afactory

332 - def unregister(self, name):

333 del self.names[name]

334

335 - def requestAvatar(self, avatarID, mind, interface):

336 assert interface == pb.IPerspective 337 afactory = self.names.get(avatarID) 338 if afactory: 339 p = afactory.getPerspective() 340 elif avatarID == "debug": 341 p = DebugPerspective() 342 p.master = self.master 343 p.botmaster = self.botmaster 344 elif avatarID == "statusClient": 345 p = self.statusClientService.getPerspective() 346 else: 347 # it must be one of the buildslaves: no other names will make it 348 # past the checker 349 p = self.botmaster.getPerspective(avatarID) 350 351 if not p: 352 raise ValueError("no perspective for '%s'" % avatarID) 353 354 d = defer.maybeDeferred(p.attached, mind) 355 d.addCallback(self._avatarAttached, mind) 356 return d

357

358 - def _avatarAttached(self, p, mind):

359 return (pb.IPerspective, p, lambda p=p,mind=mind: p.detached(mind))

360 361 ######################################## 362 363 # service hierarchy: 364 # BuildMaster 365 # BotMaster 366 # ChangeMaster 367 # all IChangeSource objects 368 # StatusClientService 369 # TCPClient(self.ircFactory) 370 # TCPServer(self.slaveFactory) -> dispatcher.requestAvatar 371 # TCPServer(self.site) 372 # UNIXServer(ResourcePublisher(self.site)) 373 374

375 -class BuildMaster(service.MultiService):

376 debug = 0 377 manhole = None 378 debugPassword = None 379 projectName = "(unspecified)" 380 projectURL = None 381 buildbotURL = None 382 change_svc = None 383 properties = Properties() 384

385 - def __init__(self, basedir, configFileName="master.cfg"):

386 service.MultiService.__init__(self) 387 self.setName("buildmaster") 388 self.basedir = basedir 389 self.configFileName = configFileName 390 391 # the dispatcher is the realm in which all inbound connections are 392 # looked up: slave builders, change notifications, status clients, and 393 # the debug port 394 dispatcher = Dispatcher() 395 dispatcher.master = self 396 self.dispatcher = dispatcher 397 self.checker = checkers.InMemoryUsernamePasswordDatabaseDontUse() 398 # the checker starts with no user/passwd pairs: they are added later 399 p = portal.Portal(dispatcher) 400 p.registerChecker(self.checker) 401 self.slaveFactory = pb.PBServerFactory(p) 402 self.slaveFactory.unsafeTracebacks = True # let them see exceptions 403 404 self.slavePortnum = None 405 self.slavePort = None 406 407 self.botmaster = BotMaster() 408 self.botmaster.setName("botmaster") 409 self.botmaster.setServiceParent(self) 410 dispatcher.botmaster = self.botmaster 411 412 self.status = Status(self.botmaster, self.basedir) 413 414 self.statusTargets = [] 415 416 # this ChangeMaster is a dummy, only used by tests. In the real 417 # buildmaster, where the BuildMaster instance is activated 418 # (startService is called) by twistd, this attribute is overwritten. 419 self.useChanges(TestChangeMaster()) 420 421 self.readConfig = False

422

423 - def startService(self):

424 service.MultiService.startService(self) 425 self.loadChanges() # must be done before loading the config file 426 if not self.readConfig: 427 # TODO: consider catching exceptions during this call to 428 # loadTheConfigFile and bailing (reactor.stop) if it fails, 429 # since without a config file we can't do anything except reload 430 # the config file, and it would be nice for the user to discover 431 # this quickly. 432 self.loadTheConfigFile() 433 if signal and hasattr(signal, "SIGHUP"): 434 signal.signal(signal.SIGHUP, self._handleSIGHUP) 435 for b in self.botmaster.builders.values(): 436 b.builder_status.addPointEvent(["master", "started"]) 437 b.builder_status.saveYourself()

438

439 - def useChanges(self, changes):

440 if self.change_svc: 441 # TODO: can return a Deferred 442 self.change_svc.disownServiceParent() 443 self.change_svc = changes 444 self.change_svc.basedir = self.basedir 445 self.change_svc.setName("changemaster") 446 self.dispatcher.changemaster = self.change_svc 447 self.change_svc.setServiceParent(self)

448

449 - def loadChanges(self):

450 filename = os.path.join(self.basedir, "changes.pck") 451 try: 452 changes = load(open(filename, "rb")) 453 styles.doUpgrade() 454 except IOError: 455 log.msg("changes.pck missing, using new one") 456 changes = ChangeMaster() 457 except EOFError: 458 log.msg("corrupted changes.pck, using new one") 459 changes = ChangeMaster() 460 self.useChanges(changes)

461

462 - def _handleSIGHUP(self, *args):

463 reactor.callLater(0, self.loadTheConfigFile)

464

465 - def getStatus(self):

466 """ 467 @rtype: L{buildbot.status.builder.Status} 468 """ 469 return self.status

470

471 - def loadTheConfigFile(self, configFile=None):

472 if not configFile: 473 configFile = os.path.join(self.basedir, self.configFileName) 474 475 log.msg("Creating BuildMaster -- buildbot.version: %s" % buildbot.version) 476 log.msg("loading configuration from %s" % configFile) 477 configFile = os.path.expanduser(configFile) 478 479 try: 480 f = open(configFile, "r") 481 except IOError, e: 482 log.msg("unable to open config file '%s'" % configFile) 483 log.msg("leaving old configuration in place") 484 log.err(e) 485 return 486 487 try: 488 self.loadConfig(f) 489 except: 490 log.msg("error during loadConfig") 491 log.err() 492 log.msg("The new config file is unusable, so I'll ignore it.") 493 log.msg("I will keep using the previous config file instead.") 494 f.close()

495

496 - def loadConfig(self, f):

497 """Internal function to load a specific configuration file. Any 498 errors in the file will be signalled by raising an exception. 499 500 @return: a Deferred that will fire (with None) when the configuration 501 changes have been completed. This may involve a round-trip to each 502 buildslave that was involved.""" 503 504 localDict = {'basedir': os.path.expanduser(self.basedir)} 505 try: 506 exec f in localDict 507 except: 508 log.msg("error while parsing config file") 509 raise 510 511 try: 512 config = localDict['BuildmasterConfig'] 513 except KeyError: 514 log.err("missing config dictionary") 515 log.err("config file must define BuildmasterConfig") 516 raise 517 518 known_keys = ("bots", "slaves", 519 "sources", "change_source", 520 "schedulers", "builders", "mergeRequests", 521 "slavePortnum", "debugPassword", "logCompressionLimit", 522 "manhole", "status", "projectName", "projectURL", 523 "buildbotURL", "properties", "prioritizeBuilders", 524 "eventHorizon", "buildCacheSize", "logHorizon", "buildHorizon", 525 "changeHorizon", "logMaxSize", "logMaxTailSize", 526 "logCompressionMethod", 527 ) 528 for k in config.keys(): 529 if k not in known_keys: 530 log.msg("unknown key '%s' defined in config dictionary" % k) 531 532 try: 533 # required 534 schedulers = config['schedulers'] 535 builders = config['builders'] 536 slavePortnum = config['slavePortnum'] 537 #slaves = config['slaves'] 538 #change_source = config['change_source'] 539 540 # optional 541 debugPassword = config.get('debugPassword') 542 manhole = config.get('manhole') 543 status = config.get('status', []) 544 projectName = config.get('projectName') 545 projectURL = config.get('projectURL') 546 buildbotURL = config.get('buildbotURL') 547 properties = config.get('properties', {}) 548 buildCacheSize = config.get('buildCacheSize', None) 549 eventHorizon = config.get('eventHorizon', None) 550 logHorizon = config.get('logHorizon', None) 551 buildHorizon = config.get('buildHorizon', None) 552 logCompressionLimit = config.get('logCompressionLimit', 4*1024) 553 if logCompressionLimit is not None and not \ 554 isinstance(logCompressionLimit, int): 555 raise ValueError("logCompressionLimit needs to be bool or int") 556 logCompressionMethod = config.get('logCompressionMethod', "bz2") 557 if logCompressionMethod not in ('bz2', 'gz'): 558 raise ValueError("logCompressionMethod needs to be 'bz2', or 'gz'") 559 logMaxSize = config.get('logMaxSize') 560 if logMaxSize is not None and not \ 561 isinstance(logMaxSize, int): 562 raise ValueError("logMaxSize needs to be None or int") 563 logMaxTailSize = config.get('logMaxTailSize') 564 if logMaxTailSize is not None and not \ 565 isinstance(logMaxTailSize, int): 566 raise ValueError("logMaxTailSize needs to be None or int") 567 mergeRequests = config.get('mergeRequests') 568 if mergeRequests is not None and not callable(mergeRequests): 569 raise ValueError("mergeRequests must be a callable") 570 prioritizeBuilders = config.get('prioritizeBuilders') 571 if prioritizeBuilders is not None and not callable(prioritizeBuilders): 572 raise ValueError("prioritizeBuilders must be callable") 573 changeHorizon = config.get("changeHorizon") 574 if changeHorizon is not None and not isinstance(changeHorizon, int): 575 raise ValueError("changeHorizon needs to be an int") 576 577 except KeyError, e: 578 log.msg("config dictionary is missing a required parameter") 579 log.msg("leaving old configuration in place") 580 raise 581 582 #if "bots" in config: 583 # raise KeyError("c['bots'] is no longer accepted") 584 585 slaves = config.get('slaves', []) 586 if "bots" in config: 587 m = ("c['bots'] is deprecated as of 0.7.6 and will be " 588 "removed by 0.8.0 . Please use c['slaves'] instead.") 589 log.msg(m) 590 warnings.warn(m, DeprecationWarning) 591 for name, passwd in config['bots']: 592 slaves.append(BuildSlave(name, passwd)) 593 594 if "bots" not in config and "slaves" not in config: 595 log.msg("config dictionary must have either 'bots' or 'slaves'") 596 log.msg("leaving old configuration in place") 597 raise KeyError("must have either 'bots' or 'slaves'") 598 599 #if "sources" in config: 600 # raise KeyError("c['sources'] is no longer accepted") 601 602 if changeHorizon is not None: 603 self.change_svc.changeHorizon = changeHorizon 604 605 change_source = config.get('change_source', []) 606 if isinstance(change_source, (list, tuple)): 607 change_sources = change_source 608 else: 609 change_sources = [change_source] 610 if "sources" in config: 611 m = ("c['sources'] is deprecated as of 0.7.6 and will be " 612 "removed by 0.8.0 . Please use c['change_source'] instead.") 613 log.msg(m) 614 warnings.warn(m, DeprecationWarning) 615 for s in config['sources']: 616 change_sources.append(s) 617 618 # do some validation first 619 for s in slaves: 620 assert interfaces.IBuildSlave.providedBy(s) 621 if s.slavename in ("debug", "change", "status"): 622 raise KeyError( 623 "reserved name '%s' used for a bot" % s.slavename) 624 if config.has_key('interlocks'): 625 raise KeyError("c['interlocks'] is no longer accepted") 626 627 assert isinstance(change_sources, (list, tuple)) 628 for s in change_sources: 629 assert interfaces.IChangeSource(s, None) 630 # this assertion catches c['schedulers'] = Scheduler(), since 631 # Schedulers are service.MultiServices and thus iterable. 632 errmsg = "c['schedulers'] must be a list of Scheduler instances" 633 assert isinstance(schedulers, (list, tuple)), errmsg 634 for s in schedulers: 635 assert interfaces.IScheduler(s, None), errmsg 636 assert isinstance(status, (list, tuple)) 637 for s in status: 638 assert interfaces.IStatusReceiver(s, None) 639 640 slavenames = [s.slavename for s in slaves] 641 buildernames = [] 642 dirnames = [] 643 644 # convert builders from objects to config dictionaries 645 builders_dicts = [] 646 for b in builders: 647 if isinstance(b, BuilderConfig): 648 builders_dicts.append(b.getConfigDict()) 649 elif type(b) is dict: 650 builders_dicts.append(b) 651 else: 652 raise ValueError("builder %s is not a BuilderConfig object (or a dict)" % b) 653 builders = builders_dicts 654 655 for b in builders: 656 if b.has_key('slavename') and b['slavename'] not in slavenames: 657 raise ValueError("builder %s uses undefined slave %s" \ 658 % (b['name'], b['slavename'])) 659 for n in b.get('slavenames', []): 660 if n not in slavenames: 661 raise ValueError("builder %s uses undefined slave %s" \ 662 % (b['name'], n)) 663 if b['name'] in buildernames: 664 raise ValueError("duplicate builder name %s" 665 % b['name']) 666 buildernames.append(b['name']) 667 668 # sanity check name (BuilderConfig does this too) 669 if b['name'].startswith("_"): 670 errmsg = ("builder names must not start with an " 671 "underscore: " + b['name']) 672 log.err(errmsg) 673 raise ValueError(errmsg) 674 675 # Fix the dictionnary with default values, in case this wasn't 676 # specified with a BuilderConfig object (which sets the same defaults) 677 b.setdefault('builddir', safeTranslate(b['name'])) 678 b.setdefault('slavebuilddir', b['builddir']) 679 680 if b['builddir'] in dirnames: 681 raise ValueError("builder %s reuses builddir %s" 682 % (b['name'], b['builddir'])) 683 dirnames.append(b['builddir']) 684 685 unscheduled_buildernames = buildernames[:] 686 schedulernames = [] 687 for s in schedulers: 688 for b in s.listBuilderNames(): 689 assert b in buildernames, \ 690 "%s uses unknown builder %s" % (s, b) 691 if b in unscheduled_buildernames: 692 unscheduled_buildernames.remove(b) 693 694 if s.name in schedulernames: 695 # TODO: schedulers share a namespace with other Service 696 # children of the BuildMaster node, like status plugins, the 697 # Manhole, the ChangeMaster, and the BotMaster (although most 698 # of these don't have names) 699 msg = ("Schedulers must have unique names, but " 700 "'%s' was a duplicate" % (s.name,)) 701 raise ValueError(msg) 702 schedulernames.append(s.name) 703 704 if unscheduled_buildernames: 705 log.msg("Warning: some Builders have no Schedulers to drive them:" 706 " %s" % (unscheduled_buildernames,)) 707 708 # assert that all locks used by the Builds and their Steps are 709 # uniquely named. 710 lock_dict = {} 711 for b in builders: 712 for l in b.get('locks', []): 713 if isinstance(l, locks.LockAccess): # User specified access to the lock 714 l = l.lockid 715 if lock_dict.has_key(l.name): 716 if lock_dict[l.name] is not l: 717 raise ValueError("Two different locks (%s and %s) " 718 "share the name %s" 719 % (l, lock_dict[l.name], l.name)) 720 else: 721 lock_dict[l.name] = l 722 # TODO: this will break with any BuildFactory that doesn't use a 723 # .steps list, but I think the verification step is more 724 # important. 725 for s in b['factory'].steps: 726 for l in s[1].get('locks', []): 727 if isinstance(l, locks.LockAccess): # User specified access to the lock 728 l = l.lockid 729 if lock_dict.has_key(l.name): 730 if lock_dict[l.name] is not l: 731 raise ValueError("Two different locks (%s and %s)" 732 " share the name %s" 733 % (l, lock_dict[l.name], l.name)) 734 else: 735 lock_dict[l.name] = l 736 737 if not isinstance(properties, dict): 738 raise ValueError("c['properties'] must be a dictionary") 739 740 # slavePortnum supposed to be a strports specification 741 if type(slavePortnum) is int: 742 slavePortnum = "tcp:%d" % slavePortnum 743 744 # now we're committed to implementing the new configuration, so do 745 # it atomically 746 # TODO: actually, this is spread across a couple of Deferreds, so it 747 # really isn't atomic. 748 749 d = defer.succeed(None) 750 751 self.projectName = projectName 752 self.projectURL = projectURL 753 self.buildbotURL = buildbotURL 754 755 self.properties = Properties() 756 self.properties.update(properties, self.configFileName) 757 758 self.status.logCompressionLimit = logCompressionLimit 759 self.status.logCompressionMethod = logCompressionMethod 760 self.status.logMaxSize = logMaxSize 761 self.status.logMaxTailSize = logMaxTailSize 762 # Update any of our existing builders with the current log parameters. 763 # This is required so that the new value is picked up after a 764 # reconfig. 765 for builder in self.botmaster.builders.values(): 766 builder.builder_status.setLogCompressionLimit(logCompressionLimit) 767 builder.builder_status.setLogCompressionMethod(logCompressionMethod) 768 builder.builder_status.setLogMaxSize(logMaxSize) 769 builder.builder_status.setLogMaxTailSize(logMaxTailSize) 770 771 if mergeRequests is not None: 772 self.botmaster.mergeRequests = mergeRequests 773 if prioritizeBuilders is not None: 774 self.botmaster.prioritizeBuilders = prioritizeBuilders 775 776 self.buildCacheSize = buildCacheSize 777 self.eventHorizon = eventHorizon 778 self.logHorizon = logHorizon 779 self.buildHorizon = buildHorizon 780 781 # self.slaves: Disconnect any that were attached and removed from the 782 # list. Update self.checker with the new list of passwords, including 783 # debug/change/status. 784 d.addCallback(lambda res: self.loadConfig_Slaves(slaves)) 785 786 # self.debugPassword 787 if debugPassword: 788 self.checker.addUser("debug", debugPassword) 789 self.debugPassword = debugPassword 790 791 # self.manhole 792 if manhole != self.manhole: 793 # changing 794 if self.manhole: 795 # disownServiceParent may return a Deferred 796 d.addCallback(lambda res: self.manhole.disownServiceParent()) 797 def _remove(res): 798 self.manhole = None 799 return res

800 d.addCallback(_remove) 801 if manhole: 802 def _add(res): 803 self.manhole = manhole 804 manhole.setServiceParent(self)

805 d.addCallback(_add) 806 807 # add/remove self.botmaster.builders to match builders. The 808 # botmaster will handle startup/shutdown issues. 809 d.addCallback(lambda res: self.loadConfig_Builders(builders)) 810 811 d.addCallback(lambda res: self.loadConfig_status(status)) 812 813 # Schedulers are added after Builders in case they start right away 814 d.addCallback(lambda res: self.loadConfig_Schedulers(schedulers)) 815 # and Sources go after Schedulers for the same reason 816 d.addCallback(lambda res: self.loadConfig_Sources(change_sources)) 817 818 # self.slavePort 819 if self.slavePortnum != slavePortnum: 820 if self.slavePort: 821 def closeSlavePort(res): 822 d1 = self.slavePort.disownServiceParent() 823 self.slavePort = None 824 return d1 825 d.addCallback(closeSlavePort) 826 if slavePortnum is not None: 827 def openSlavePort(res): 828 self.slavePort = strports.service(slavePortnum, 829 self.slaveFactory) 830 self.slavePort.setServiceParent(self) 831 d.addCallback(openSlavePort) 832 log.msg("BuildMaster listening on port %s" % slavePortnum) 833 self.slavePortnum = slavePortnum 834 835 log.msg("configuration update started") 836 def _done(res): 837 self.readConfig = True 838 log.msg("configuration update complete") 839 d.addCallback(_done) 840 d.addCallback(lambda res: self.botmaster.maybeStartAllBuilds()) 841 return d 842

843 - def loadConfig_Slaves(self, new_slaves):

844 # set up the Checker with the names and passwords of all valid bots 845 self.checker.users = {} # violates abstraction, oh well 846 for s in new_slaves: 847 self.checker.addUser(s.slavename, s.password) 848 self.checker.addUser("change", "changepw") 849 # let the BotMaster take care of the rest 850 return self.botmaster.loadConfig_Slaves(new_slaves)

851

852 - def loadConfig_Sources(self, sources):

853 if not sources: 854 log.msg("warning: no ChangeSources specified in c['change_source']") 855 # shut down any that were removed, start any that were added 856 deleted_sources = [s for s in self.change_svc if s not in sources] 857 added_sources = [s for s in sources if s not in self.change_svc] 858 log.msg("adding %d new changesources, removing %d" % 859 (len(added_sources), len(deleted_sources))) 860 dl = [self.change_svc.removeSource(s) for s in deleted_sources] 861 def addNewOnes(res): 862 [self.change_svc.addSource(s) for s in added_sources]

863 d = defer.DeferredList(dl, fireOnOneErrback=1, consumeErrors=0) 864 d.addCallback(addNewOnes) 865 return d 866

867 - def allSchedulers(self):

868 return [child for child in self 869 if interfaces.IScheduler.providedBy(child)]

870 871

872 - def loadConfig_Schedulers(self, newschedulers):

873 oldschedulers = self.allSchedulers() 874 removed = [s for s in oldschedulers if s not in newschedulers] 875 added = [s for s in newschedulers if s not in oldschedulers] 876 dl = [defer.maybeDeferred(s.disownServiceParent) for s in removed] 877 def addNewOnes(res): 878 log.msg("adding %d new schedulers, removed %d" % 879 (len(added), len(dl))) 880 for s in added: 881 s.setServiceParent(self)

882 d = defer.DeferredList(dl, fireOnOneErrback=1) 883 d.addCallback(addNewOnes) 884 if removed or added: 885 # notify Downstream schedulers to potentially pick up 886 # new schedulers now that we have removed and added some 887 def updateDownstreams(res): 888 log.msg("notifying downstream schedulers of changes") 889 for s in newschedulers: 890 if interfaces.IDownstreamScheduler.providedBy(s): 891 s.checkUpstreamScheduler() 892 d.addCallback(updateDownstreams) 893 return d 894

895 - def loadConfig_Builders(self, newBuilderData):

896 somethingChanged = False 897 newList = {} 898 newBuilderNames = [] 899 allBuilders = self.botmaster.builders.copy() 900 for data in newBuilderData: 901 name = data['name'] 902 newList[name] = data 903 newBuilderNames.append(name) 904 905 # identify all that were removed 906 for oldname in self.botmaster.getBuildernames(): 907 if oldname not in newList: 908 log.msg("removing old builder %s" % oldname) 909 del allBuilders[oldname] 910 somethingChanged = True 911 # announce the change 912 self.status.builderRemoved(oldname) 913 914 # everything in newList is either unchanged, changed, or new 915 for name, data in newList.items(): 916 old = self.botmaster.builders.get(name) 917 basedir = data['builddir'] 918 #name, slave, builddir, factory = data 919 if not old: # new 920 # category added after 0.6.2 921 category = data.get('category', None) 922 log.msg("adding new builder %s for category %s" % 923 (name, category)) 924 statusbag = self.status.builderAdded(name, basedir, category) 925 builder = Builder(data, statusbag) 926 allBuilders[name] = builder 927 somethingChanged = True 928 elif old.compareToSetup(data): 929 # changed: try to minimize the disruption and only modify the 930 # pieces that really changed 931 diffs = old.compareToSetup(data) 932 log.msg("updating builder %s: %s" % (name, "\n".join(diffs))) 933 934 statusbag = old.builder_status 935 statusbag.saveYourself() # seems like a good idea 936 # TODO: if the basedir was changed, we probably need to make 937 # a new statusbag 938 new_builder = Builder(data, statusbag) 939 new_builder.consumeTheSoulOfYourPredecessor(old) 940 # that migrates any retained slavebuilders too 941 942 # point out that the builder was updated. On the Waterfall, 943 # this will appear just after any currently-running builds. 944 statusbag.addPointEvent(["config", "updated"]) 945 946 allBuilders[name] = new_builder 947 somethingChanged = True 948 else: 949 # unchanged: leave it alone 950 log.msg("builder %s is unchanged" % name) 951 pass 952 953 # regardless of whether anything changed, get each builder status 954 # to update its config 955 for builder in allBuilders.values(): 956 builder.builder_status.reconfigFromBuildmaster(self) 957 958 # and then tell the botmaster if anything's changed 959 if somethingChanged: 960 sortedAllBuilders = [allBuilders[name] for name in newBuilderNames] 961 d = self.botmaster.setBuilders(sortedAllBuilders) 962 return d 963 return None

964

965 - def loadConfig_status(self, status):

966 dl = [] 967 968 # remove old ones 969 for s in self.statusTargets[:]: 970 if not s in status: 971 log.msg("removing IStatusReceiver", s) 972 d = defer.maybeDeferred(s.disownServiceParent) 973 dl.append(d) 974 self.statusTargets.remove(s) 975 # after those are finished going away, add new ones 976 def addNewOnes(res): 977 for s in status: 978 if not s in self.statusTargets: 979 log.msg("adding IStatusReceiver", s) 980 s.setServiceParent(self) 981 self.statusTargets.append(s)

982 d = defer.DeferredList(dl, fireOnOneErrback=1) 983 d.addCallback(addNewOnes) 984 return d 985 986

987 - def addChange(self, change):

988 for s in self.allSchedulers(): 989 s.addChange(change)

990

991 - def submitBuildSet(self, bs):

992 # determine the set of Builders to use 993 builders = [] 994 for name in bs.builderNames: 995 b = self.botmaster.builders.get(name) 996 if b: 997 if b not in builders: 998 builders.append(b) 999 continue 1000 # TODO: add aliases like 'all' 1001 raise KeyError("no such builder named '%s'" % name) 1002 1003 # now tell the BuildSet to create BuildRequests for all those 1004 # Builders and submit them 1005 bs.start(builders) 1006 self.status.buildsetSubmitted(bs.status)

1007 1008

1009 -class Control:

1010 implements(interfaces.IControl) 1011

1012 - def __init__(self, master):

1013 self.master = master

1014

1015 - def addChange(self, change):

1016 self.master.change_svc.addChange(change)

1017

1018 - def submitBuildSet(self, bs):

1019 self.master.submitBuildSet(bs)

1020

1021 - def getBuilder(self, name):

1022 b = self.master.botmaster.builders[name] 1023 return interfaces.IBuilderControl(b)

1024 1025 components.registerAdapter(Control, BuildMaster, interfaces.IControl) 1026 1027 # so anybody who can get a handle on the BuildMaster can cause a build with: 1028 # IControl(master).getBuilder("full-2.3").requestBuild(buildrequest) 1029

Source Code for Module buildbot.master