buildbot.master

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 17 import re 18 import os 19 import signal 20 import textwrap 21 import socket 22 23 from zope.interface import implements 24 from twisted.python import log, components 25 from twisted.internet import defer, reactor 26 from twisted.application import service 27 from twisted.application.internet import TimerService 28 29 import buildbot 30 import buildbot.pbmanager 31 from buildbot.util import safeTranslate, subscription, epoch2datetime 32 from buildbot.process.builder import Builder 33 from buildbot.status.master import Status 34 from buildbot.changes import changes 35 from buildbot.changes.manager import ChangeManager 36 from buildbot import interfaces, locks 37 from buildbot.process.properties import Properties 38 from buildbot.config import BuilderConfig, MasterConfig 39 from buildbot.process.builder import BuilderControl 40 from buildbot.db import connector, exceptions 41 from buildbot.schedulers.manager import SchedulerManager 42 from buildbot.schedulers.base import isScheduler 43 from buildbot.process.botmaster import BotMaster 44 from buildbot.process import debug 45 from buildbot.process import metrics 46 from buildbot.process import cache 47 from buildbot.status.results import SUCCESS, WARNINGS, FAILURE 48 from buildbot import monkeypatches

49 50 ######################################## 51 52 -class _Unset: pass # marker

53

54 -class LogRotation:

55 '''holds log rotation parameters (for WebStatus)'''

56 - def __init__(self):

57 self.rotateLength = 1 * 1000 * 1000 58 self.maxRotatedFiles = 10

59

60 -class BuildMaster(service.MultiService):

61 debug = 0 62 manhole = None 63 debugPassword = None 64 title = "(unspecified)" 65 titleURL = None 66 buildbotURL = None 67 change_svc = None 68 properties = Properties() 69 70 # frequency with which to reclaim running builds; this should be set to 71 # something fairly long, to avoid undue database load 72 RECLAIM_BUILD_INTERVAL = 10*60 73 74 # multiplier on RECLAIM_BUILD_INTERVAL at which a build is considered 75 # unclaimed; this should be at least 2 to avoid false positives 76 UNCLAIMED_BUILD_FACTOR = 6 77 78 # if this quantity of unclaimed build requests are present in the table, 79 # then something is probably wrong! The master will log a WARNING on every 80 # database poll operation. 81 WARNING_UNCLAIMED_COUNT = 10000 82

83 - def __init__(self, basedir, configFileName="master.cfg"):

84 service.MultiService.__init__(self) 85 self.setName("buildmaster") 86 self.basedir = basedir 87 assert os.path.isdir(self.basedir) 88 self.configFileName = configFileName 89 90 self.pbmanager = buildbot.pbmanager.PBManager() 91 self.pbmanager.setServiceParent(self) 92 "L{buildbot.pbmanager.PBManager} instance managing connections for this master" 93 94 self.slavePortnum = None 95 self.slavePort = None 96 97 self.change_svc = ChangeManager() 98 self.change_svc.setServiceParent(self) 99 100 self.botmaster = BotMaster(self) 101 self.botmaster.setName("botmaster") 102 self.botmaster.setServiceParent(self) 103 104 self.scheduler_manager = SchedulerManager(self) 105 self.scheduler_manager.setName('scheduler_manager') 106 self.scheduler_manager.setServiceParent(self) 107 108 self.caches = cache.CacheManager() 109 110 self.debugClientRegistration = None 111 112 self.statusTargets = [] 113 114 self.config = MasterConfig() 115 116 self.db = None 117 self.db_url = None 118 self.db_poll_interval = _Unset 119 120 self.metrics = None 121 122 # note that "read" here is taken in the past participal (i.e., "I read 123 # the config already") rather than the imperative ("you should read the 124 # config later") 125 self.readConfig = False 126 127 # create log_rotation object and set default parameters (used by WebStatus) 128 self.log_rotation = LogRotation() 129 130 # subscription points 131 self._change_subs = \ 132 subscription.SubscriptionPoint("changes") 133 self._new_buildrequest_subs = \ 134 subscription.SubscriptionPoint("buildrequest_additions") 135 self._new_buildset_subs = \ 136 subscription.SubscriptionPoint("buildset_additions") 137 self._complete_buildset_subs = \ 138 subscription.SubscriptionPoint("buildset_completion") 139 140 # set up the tip of the status hierarchy (must occur after subscription 141 # points are initialized) 142 self.status = Status(self) 143 144 # local cache for this master's object ID 145 self._object_id = None

146

147 - def startService(self):

148 # first, apply all monkeypatches 149 monkeypatches.patch_all() 150 151 service.MultiService.startService(self) 152 if not self.readConfig: 153 # TODO: consider catching exceptions during this call to 154 # loadTheConfigFile and bailing (reactor.stop) if it fails, 155 # since without a config file we can't do anything except reload 156 # the config file, and it would be nice for the user to discover 157 # this quickly. 158 self.loadTheConfigFile() 159 if hasattr(signal, "SIGHUP"): 160 signal.signal(signal.SIGHUP, self._handleSIGHUP) 161 for b in self.botmaster.builders.values(): 162 b.builder_status.addPointEvent(["master", "started"]) 163 b.builder_status.saveYourself()

164

165 - def _handleSIGHUP(self, *args):

166 reactor.callLater(0, self.loadTheConfigFile)

167

168 - def getStatus(self):

169 """ 170 @rtype: L{buildbot.status.builder.Status} 171 """ 172 return self.status

173

174 - def loadTheConfigFile(self, configFile=None):

175 if not configFile: 176 configFile = os.path.join(self.basedir, self.configFileName) 177 178 log.msg("Creating BuildMaster -- buildbot.version: %s" % buildbot.version) 179 log.msg("loading configuration from %s" % configFile) 180 configFile = os.path.expanduser(configFile) 181 182 try: 183 f = open(configFile, "r") 184 except IOError, e: 185 log.msg("unable to open config file '%s'" % configFile) 186 log.msg("leaving old configuration in place") 187 log.err(e) 188 return 189 190 try: 191 d = self.loadConfig(f) 192 except: 193 log.msg("error during loadConfig") 194 log.err() 195 log.msg("The new config file is unusable, so I'll ignore it.") 196 log.msg("I will keep using the previous config file instead.") 197 return # sorry unit tests 198 f.close() 199 return d # for unit tests

200

201 - def loadConfig(self, f, checkOnly=False):

202 """Internal function to load a specific configuration file. Any 203 errors in the file will be signalled by raising a failure. Returns 204 a deferred. 205 """ 206 207 # this entire operation executes in a deferred, so that any exceptions 208 # are automatically converted to a failure object. 209 d = defer.succeed(None) 210 211 def do_load(_): 212 log.msg("configuration update started") 213 214 # execute the config file 215 216 localDict = {'basedir': os.path.expanduser(self.basedir), 217 '__file__': os.path.abspath(self.configFileName)} 218 219 try: 220 exec f in localDict 221 except: 222 log.msg("error while parsing config file") 223 raise 224 225 try: 226 config = localDict['BuildmasterConfig'] 227 except KeyError: 228 log.err("missing config dictionary") 229 log.err("config file must define BuildmasterConfig") 230 raise 231 232 # check for unknown keys 233 234 known_keys = ("slaves", "change_source", 235 "schedulers", "builders", "mergeRequests", 236 "slavePortnum", "debugPassword", "logCompressionLimit", 237 "manhole", "status", "projectName", "projectURL", 238 "title", "titleURL", 239 "buildbotURL", "properties", "prioritizeBuilders", 240 "eventHorizon", "buildCacheSize", "changeCacheSize", 241 "logHorizon", "buildHorizon", "changeHorizon", 242 "logMaxSize", "logMaxTailSize", "logCompressionMethod", 243 "db_url", "multiMaster", "db_poll_interval", 244 "metrics", "caches" 245 ) 246 for k in config.keys(): 247 if k not in known_keys: 248 log.msg("unknown key '%s' defined in config dictionary" % k) 249 250 # load known keys into local vars, applying defaults 251 252 try: 253 # required 254 schedulers = config['schedulers'] 255 builders = config['builders'] 256 slavePortnum = config['slavePortnum'] 257 #slaves = config['slaves'] 258 #change_source = config['change_source'] 259 260 # optional 261 db_url = config.get("db_url", "sqlite:///state.sqlite") 262 db_poll_interval = config.get("db_poll_interval", None) 263 debugPassword = config.get('debugPassword') 264 manhole = config.get('manhole') 265 status = config.get('status', []) 266 # projectName/projectURL still supported to avoid 267 # breaking legacy configurations 268 title = config.get('title', config.get('projectName')) 269 titleURL = config.get('titleURL', config.get('projectURL')) 270 buildbotURL = config.get('buildbotURL') 271 properties = config.get('properties', {}) 272 buildCacheSize = config.get('buildCacheSize', None) 273 changeCacheSize = config.get('changeCacheSize', None) 274 eventHorizon = config.get('eventHorizon', 50) 275 logHorizon = config.get('logHorizon', None) 276 buildHorizon = config.get('buildHorizon', None) 277 logCompressionLimit = config.get('logCompressionLimit', 4*1024) 278 if logCompressionLimit is not None and not \ 279 isinstance(logCompressionLimit, int): 280 raise ValueError("logCompressionLimit needs to be bool or int") 281 logCompressionMethod = config.get('logCompressionMethod', "bz2") 282 if logCompressionMethod not in ('bz2', 'gz'): 283 raise ValueError("logCompressionMethod needs to be 'bz2', or 'gz'") 284 logMaxSize = config.get('logMaxSize') 285 if logMaxSize is not None and not \ 286 isinstance(logMaxSize, int): 287 raise ValueError("logMaxSize needs to be None or int") 288 logMaxTailSize = config.get('logMaxTailSize') 289 if logMaxTailSize is not None and not \ 290 isinstance(logMaxTailSize, int): 291 raise ValueError("logMaxTailSize needs to be None or int") 292 mergeRequests = config.get('mergeRequests') 293 if (mergeRequests not in (None, True, False) 294 and not callable(mergeRequests)): 295 raise ValueError("mergeRequests must be a callable or False") 296 prioritizeBuilders = config.get('prioritizeBuilders') 297 if prioritizeBuilders is not None and not callable(prioritizeBuilders): 298 raise ValueError("prioritizeBuilders must be callable") 299 changeHorizon = config.get("changeHorizon") 300 if changeHorizon is not None and not isinstance(changeHorizon, int): 301 raise ValueError("changeHorizon needs to be an int") 302 303 multiMaster = config.get("multiMaster", False) 304 305 metrics_config = config.get("metrics") 306 caches_config = config.get("caches", {}) 307 308 # load validation, with defaults, and verify no unrecognized 309 # keys are included. 310 validation_defaults = { 311 'branch' : re.compile(r'^[\w.+/~-]*$'), 312 'revision' : re.compile(r'^[ \w\.\-\/]*$'), 313 'property_name' : re.compile(r'^[\w\.\-\/\~:]*$'), 314 'property_value' : re.compile(r'^[\w\.\-\/\~:]*$'), 315 } 316 validation_config = validation_defaults.copy() 317 validation_config.update(config.get("validation", {})) 318 v_config_keys = set(validation_config.keys()) 319 v_default_keys = set(validation_defaults.keys()) 320 if v_config_keys > v_default_keys: 321 raise ValueError("unrecognized validation key(s): %s" % 322 (", ".join(v_config_keys - v_default_keys,))) 323 324 except KeyError: 325 log.msg("config dictionary is missing a required parameter") 326 log.msg("leaving old configuration in place") 327 raise 328 329 if "sources" in config: 330 m = ("c['sources'] is deprecated as of 0.7.6 and is no longer " 331 "accepted in >= 0.8.0 . Please use c['change_source'] instead.") 332 raise KeyError(m) 333 334 if "bots" in config: 335 m = ("c['bots'] is deprecated as of 0.7.6 and is no longer " 336 "accepted in >= 0.8.0 . Please use c['slaves'] instead.") 337 raise KeyError(m) 338 339 # Set up metrics and caches 340 self.loadConfig_Metrics(metrics_config) 341 self.loadConfig_Caches(caches_config, buildCacheSize, 342 changeCacheSize) 343 344 slaves = config.get('slaves', []) 345 if "slaves" not in config: 346 log.msg("config dictionary must have a 'slaves' key") 347 log.msg("leaving old configuration in place") 348 raise KeyError("must have a 'slaves' key") 349 350 self.config.changeHorizon = changeHorizon 351 self.config.validation = validation_config 352 353 change_source = config.get('change_source', []) 354 if isinstance(change_source, (list, tuple)): 355 change_sources = change_source 356 else: 357 change_sources = [change_source] 358 359 # do some validation first 360 for s in slaves: 361 assert interfaces.IBuildSlave.providedBy(s) 362 if s.slavename in ("debug", "change", "status"): 363 raise KeyError( 364 "reserved name '%s' used for a bot" % s.slavename) 365 if config.has_key('interlocks'): 366 raise KeyError("c['interlocks'] is no longer accepted") 367 assert self.db_url is None or db_url == self.db_url, \ 368 "Cannot change db_url after master has started" 369 assert db_poll_interval is None or isinstance(db_poll_interval, int), \ 370 "db_poll_interval must be an integer: seconds between polls" 371 assert self.db_poll_interval is _Unset or db_poll_interval == self.db_poll_interval, \ 372 "Cannot change db_poll_interval after master has started" 373 374 assert isinstance(change_sources, (list, tuple)) 375 for s in change_sources: 376 assert interfaces.IChangeSource(s, None) 377 self.checkConfig_Schedulers(schedulers) 378 assert isinstance(status, (list, tuple)) 379 for s in status: 380 assert interfaces.IStatusReceiver(s, None) 381 382 slavenames = [s.slavename for s in slaves] 383 buildernames = [] 384 dirnames = [] 385 386 # convert builders from objects to config dictionaries 387 builders_dicts = [] 388 for b in builders: 389 if isinstance(b, BuilderConfig): 390 builders_dicts.append(b.getConfigDict()) 391 elif type(b) is dict: 392 builders_dicts.append(b) 393 else: 394 raise ValueError("builder %s is not a BuilderConfig object (or a dict)" % b) 395 builders = builders_dicts 396 397 for b in builders: 398 if b.has_key('slavename') and b['slavename'] not in slavenames: 399 raise ValueError("builder %s uses undefined slave %s" \ 400 % (b['name'], b['slavename'])) 401 for n in b.get('slavenames', []): 402 if n not in slavenames: 403 raise ValueError("builder %s uses undefined slave %s" \ 404 % (b['name'], n)) 405 if b['name'] in buildernames: 406 raise ValueError("duplicate builder name %s" 407 % b['name']) 408 buildernames.append(b['name']) 409 410 # sanity check name (BuilderConfig does this too) 411 if b['name'].startswith("_"): 412 errmsg = ("builder names must not start with an " 413 "underscore: " + b['name']) 414 log.err(errmsg) 415 raise ValueError(errmsg) 416 417 # Fix the dictionary with default values, in case this wasn't 418 # specified with a BuilderConfig object (which sets the same defaults) 419 b.setdefault('builddir', safeTranslate(b['name'])) 420 b.setdefault('slavebuilddir', b['builddir']) 421 b.setdefault('buildHorizon', buildHorizon) 422 b.setdefault('logHorizon', logHorizon) 423 b.setdefault('eventHorizon', eventHorizon) 424 if b['builddir'] in dirnames: 425 raise ValueError("builder %s reuses builddir %s" 426 % (b['name'], b['builddir'])) 427 dirnames.append(b['builddir']) 428 429 unscheduled_buildernames = buildernames[:] 430 schedulernames = [] 431 for s in schedulers: 432 for b in s.listBuilderNames(): 433 # Skip checks for builders in multimaster mode 434 if not multiMaster: 435 assert b in buildernames, \ 436 "%s uses unknown builder %s" % (s, b) 437 if b in unscheduled_buildernames: 438 unscheduled_buildernames.remove(b) 439 440 if s.name in schedulernames: 441 msg = ("Schedulers must have unique names, but " 442 "'%s' was a duplicate" % (s.name,)) 443 raise ValueError(msg) 444 schedulernames.append(s.name) 445 446 # Skip the checks for builders in multimaster mode 447 if not multiMaster and unscheduled_buildernames: 448 log.msg("Warning: some Builders have no Schedulers to drive them:" 449 " %s" % (unscheduled_buildernames,)) 450 451 # assert that all locks used by the Builds and their Steps are 452 # uniquely named. 453 lock_dict = {} 454 for b in builders: 455 for l in b.get('locks', []): 456 if isinstance(l, locks.LockAccess): # User specified access to the lock 457 l = l.lockid 458 if lock_dict.has_key(l.name): 459 if lock_dict[l.name] is not l: 460 raise ValueError("Two different locks (%s and %s) " 461 "share the name %s" 462 % (l, lock_dict[l.name], l.name)) 463 else: 464 lock_dict[l.name] = l 465 # TODO: this will break with any BuildFactory that doesn't use a 466 # .steps list, but I think the verification step is more 467 # important. 468 for s in b['factory'].steps: 469 for l in s[1].get('locks', []): 470 if isinstance(l, locks.LockAccess): # User specified access to the lock 471 l = l.lockid 472 if lock_dict.has_key(l.name): 473 if lock_dict[l.name] is not l: 474 raise ValueError("Two different locks (%s and %s)" 475 " share the name %s" 476 % (l, lock_dict[l.name], l.name)) 477 else: 478 lock_dict[l.name] = l 479 480 if not isinstance(properties, dict): 481 raise ValueError("c['properties'] must be a dictionary") 482 483 # slavePortnum supposed to be a strports specification 484 if type(slavePortnum) is int: 485 slavePortnum = "tcp:%d" % slavePortnum 486 487 ### ---- everything from here on down is done only on an actual (re)start 488 if checkOnly: 489 return config 490 491 self.title = title 492 self.titleURL = titleURL 493 self.buildbotURL = buildbotURL 494 495 self.properties = Properties() 496 self.properties.update(properties, self.configFileName) 497 498 self.status.logCompressionLimit = logCompressionLimit 499 self.status.logCompressionMethod = logCompressionMethod 500 self.status.logMaxSize = logMaxSize 501 self.status.logMaxTailSize = logMaxTailSize 502 # Update any of our existing builders with the current log parameters. 503 # This is required so that the new value is picked up after a 504 # reconfig. 505 for builder in self.botmaster.builders.values(): 506 builder.builder_status.setLogCompressionLimit(logCompressionLimit) 507 builder.builder_status.setLogCompressionMethod(logCompressionMethod) 508 builder.builder_status.setLogMaxSize(logMaxSize) 509 builder.builder_status.setLogMaxTailSize(logMaxTailSize) 510 511 if mergeRequests is not None: 512 self.botmaster.mergeRequests = mergeRequests 513 if prioritizeBuilders is not None: 514 self.botmaster.prioritizeBuilders = prioritizeBuilders 515 516 self.buildCacheSize = buildCacheSize 517 self.changeCacheSize = changeCacheSize 518 self.eventHorizon = eventHorizon 519 self.logHorizon = logHorizon 520 self.buildHorizon = buildHorizon 521 self.slavePortnum = slavePortnum # TODO: move this to master.config.slavePortnum 522 523 # Set up the database 524 d.addCallback(lambda res: 525 self.loadConfig_Database(db_url, db_poll_interval)) 526 527 # set up slaves 528 d.addCallback(lambda res: self.loadConfig_Slaves(slaves)) 529 530 # self.manhole 531 if manhole != self.manhole: 532 # changing 533 if self.manhole: 534 # disownServiceParent may return a Deferred 535 d.addCallback(lambda res: self.manhole.disownServiceParent()) 536 def _remove(res): 537 self.manhole = None 538 return res

539 d.addCallback(_remove) 540 if manhole: 541 def _add(res): 542 self.manhole = manhole 543 manhole.setServiceParent(self)

544 d.addCallback(_add) 545 546 # add/remove self.botmaster.builders to match builders. The 547 # botmaster will handle startup/shutdown issues. 548 d.addCallback(lambda res: self.loadConfig_Builders(builders)) 549 550 d.addCallback(lambda res: self.loadConfig_Status(status)) 551 552 # Schedulers are added after Builders in case they start right away 553 d.addCallback(lambda _: self.loadConfig_Schedulers(schedulers)) 554 555 # and Sources go after Schedulers for the same reason 556 d.addCallback(lambda res: self.loadConfig_Sources(change_sources)) 557 558 # debug client 559 d.addCallback(lambda res: self.loadConfig_DebugClient(debugPassword)) 560 561 d.addCallback(do_load) 562 563 def _done(res): 564 self.readConfig = True 565 log.msg("configuration update complete") 566 # the remainder is only done if we are really loading the config 567 if not checkOnly: 568 d.addCallback(_done) 569 d.addErrback(log.err) 570 return d 571

572 - def loadConfig_Metrics(self, metrics_config):

573 if metrics_config: 574 if self.metrics: 575 self.metrics.reloadConfig(metrics_config) 576 else: 577 self.metrics = metrics.MetricLogObserver(metrics_config) 578 self.metrics.setServiceParent(self) 579 580 metrics.MetricCountEvent.log("loaded_config", 1) 581 else: 582 if self.metrics: 583 self.metrics.disownServiceParent() 584 self.metrics = None

585

586 - def loadConfig_Caches(self, caches_config, buildCacheSize, 587 changeCacheSize):

588 if buildCacheSize is not None: 589 caches_config['builds'] = buildCacheSize 590 if changeCacheSize is not None: 591 caches_config['changes'] = changeCacheSize 592 self.caches.load_config(caches_config)

593

594 - def loadDatabase(self, db_url, db_poll_interval=None):

595 if self.db: 596 return 597 598 self.db = connector.DBConnector(self, db_url, self.basedir) 599 self.db.setServiceParent(self) 600 601 # make sure it's up to date 602 d = self.db.model.is_current() 603 def check_current(res): 604 if res: 605 return # good to go! 606 raise exceptions.DatabaseNotReadyError, textwrap.dedent(""" 607 The Buildmaster database needs to be upgraded before this version of buildbot 608 can run. Use the following command-line 609 buildbot upgrade-master path/to/master 610 to upgrade the database, and try starting the buildmaster again. You may want 611 to make a backup of your buildmaster before doing so. If you are using MySQL, 612 you must specify the connector string on the upgrade-master command line: 613 buildbot upgrade-master --db=<db-url> path/to/master 614 """)

615 d.addCallback(check_current) 616 617 # set up the stuff that depends on the db 618 def set_up_db_dependents(r): 619 # subscribe the various parts of the system to changes 620 self._change_subs.subscribe(self.status.changeAdded) 621 622 # Set db_poll_interval (perhaps to 30 seconds) if you are using 623 # multiple buildmasters that share a common database, such that the 624 # masters need to discover what each other is doing by polling the 625 # database. 626 if db_poll_interval: 627 t1 = TimerService(db_poll_interval, self.pollDatabase) 628 t1.setServiceParent(self) 629 # adding schedulers (like when loadConfig happens) will trigger the 630 # scheduler loop at least once, which we need to jump-start things 631 # like Periodic. 632 d.addCallback(set_up_db_dependents) 633 return d 634

635 - def loadConfig_Database(self, db_url, db_poll_interval):

636 self.db_url = db_url 637 self.db_poll_interval = db_poll_interval 638 return self.loadDatabase(db_url, db_poll_interval)

639

640 - def loadConfig_Slaves(self, new_slaves):

641 return self.botmaster.loadConfig_Slaves(new_slaves)

642

643 - def loadConfig_Sources(self, sources):

644 timer = metrics.Timer("BuildMaster.loadConfig_Sources()") 645 timer.start() 646 if not sources: 647 log.msg("warning: no ChangeSources specified in c['change_source']") 648 # shut down any that were removed, start any that were added 649 deleted_sources = [s for s in self.change_svc if s not in sources] 650 added_sources = [s for s in sources if s not in self.change_svc] 651 log.msg("adding %d new changesources, removing %d" % 652 (len(added_sources), len(deleted_sources))) 653 dl = [self.change_svc.removeSource(s) for s in deleted_sources] 654 def addNewOnes(res): 655 [self.change_svc.addSource(s) for s in added_sources]

656 d = defer.DeferredList(dl, fireOnOneErrback=1, consumeErrors=0) 657 d.addCallback(addNewOnes) 658 659 def logCount(_): 660 timer.stop() 661 metrics.MetricCountEvent.log("num_sources", 662 len(list(self.change_svc)), absolute=True) 663 return _ 664 d.addBoth(logCount) 665 return d 666

667 - def loadConfig_DebugClient(self, debugPassword):

668 # unregister the old name.. 669 if self.debugClientRegistration: 670 d = self.debugClientRegistration.unregister() 671 self.debugClientRegistration = None 672 else: 673 d = defer.succeed(None) 674 675 # and register the new one 676 def reg(_): 677 if debugPassword: 678 self.debugClientRegistration = debug.registerDebugClient( 679 self, self.slavePortnum, debugPassword, self.pbmanager)

680 d.addCallback(reg) 681 return d 682

683 - def allSchedulers(self):

684 return list(self.scheduler_manager)

685

686 - def loadConfig_Builders(self, newBuilderData):

687 timer = metrics.Timer("BuildMaster.loadConfig_Builders()") 688 timer.start() 689 somethingChanged = False 690 newList = {} 691 newBuilderNames = [] 692 allBuilders = self.botmaster.builders.copy() 693 for data in newBuilderData: 694 name = data['name'] 695 newList[name] = data 696 newBuilderNames.append(name) 697 698 # identify all that were removed 699 for oldname in self.botmaster.getBuildernames(): 700 if oldname not in newList: 701 log.msg("removing old builder %s" % oldname) 702 del allBuilders[oldname] 703 somethingChanged = True 704 # announce the change 705 self.status.builderRemoved(oldname) 706 707 # everything in newList is either unchanged, changed, or new 708 for name, data in newList.items(): 709 old = self.botmaster.builders.get(name) 710 basedir = data['builddir'] 711 #name, slave, builddir, factory = data 712 if not old: # new 713 # category added after 0.6.2 714 category = data.get('category', None) 715 log.msg("adding new builder %s for category %s" % 716 (name, category)) 717 statusbag = self.status.builderAdded(name, basedir, category) 718 builder = Builder(data, statusbag) 719 allBuilders[name] = builder 720 somethingChanged = True 721 elif old.compareToSetup(data): 722 # changed: try to minimize the disruption and only modify the 723 # pieces that really changed 724 diffs = old.compareToSetup(data) 725 log.msg("updating builder %s: %s" % (name, "\n".join(diffs))) 726 727 statusbag = old.builder_status 728 statusbag.saveYourself() # seems like a good idea 729 # TODO: if the basedir was changed, we probably need to make 730 # a new statusbag 731 new_builder = Builder(data, statusbag) 732 new_builder.consumeTheSoulOfYourPredecessor(old) 733 # that migrates any retained slavebuilders too 734 735 # point out that the builder was updated. On the Waterfall, 736 # this will appear just after any currently-running builds. 737 statusbag.addPointEvent(["config", "updated"]) 738 739 allBuilders[name] = new_builder 740 somethingChanged = True 741 else: 742 # unchanged: leave it alone 743 log.msg("builder %s is unchanged" % name) 744 pass 745 746 # regardless of whether anything changed, get each builder status 747 # to update its config 748 for builder in allBuilders.values(): 749 builder.builder_status.reconfigFromBuildmaster(self) 750 751 metrics.MetricCountEvent.log("num_builders", 752 len(allBuilders), absolute=True) 753 754 # and then tell the botmaster if anything's changed 755 if somethingChanged: 756 sortedAllBuilders = [allBuilders[name] for name in newBuilderNames] 757 d = self.botmaster.setBuilders(sortedAllBuilders) 758 def stop_timer(_): 759 timer.stop() 760 return _

761 d.addBoth(stop_timer) 762 return d 763 764 return None 765

766 - def loadConfig_Status(self, status):

767 timer = metrics.Timer("BuildMaster.loadConfig_Status()") 768 timer.start() 769 dl = [] 770 771 # remove old ones 772 for s in self.statusTargets[:]: 773 if not s in status: 774 log.msg("removing IStatusReceiver", s) 775 d = defer.maybeDeferred(s.disownServiceParent) 776 dl.append(d) 777 self.statusTargets.remove(s) 778 # after those are finished going away, add new ones 779 def addNewOnes(res): 780 for s in status: 781 if not s in self.statusTargets: 782 log.msg("adding IStatusReceiver", s) 783 s.setServiceParent(self) 784 self.statusTargets.append(s)

785 d = defer.DeferredList(dl, fireOnOneErrback=1) 786 d.addCallback(addNewOnes) 787 788 def logCount(_): 789 timer.stop() 790 metrics.MetricCountEvent.log("num_status", 791 len(self.statusTargets), absolute=True) 792 return _ 793 d.addBoth(logCount) 794 795 return d 796

797 - def checkConfig_Schedulers(self, schedulers):

798 # this assertion catches c['schedulers'] = Scheduler(), since 799 # Schedulers are service.MultiServices and thus iterable. 800 errmsg = "c['schedulers'] must be a list of Scheduler instances" 801 assert isinstance(schedulers, (list, tuple)), errmsg 802 for s in schedulers: 803 assert isScheduler(s), errmsg

804

805 - def loadConfig_Schedulers(self, schedulers):

806 timer = metrics.Timer("BuildMaster.loadConfig_Schedulers()") 807 timer.start() 808 d = self.scheduler_manager.updateSchedulers(schedulers) 809 def logCount(_): 810 timer.stop() 811 metrics.MetricCountEvent.log("num_schedulers", 812 len(list(self.scheduler_manager)), absolute=True) 813 return _

814 d.addBoth(logCount) 815 return d 816 817 ## informational methods 818

819 - def getObjectId(self):

820 """ 821 Return the obejct id for this master, for associating state with the master. 822 823 @returns: ID, via Deferred 824 """ 825 # try to get the cached value 826 if self._object_id is not None: 827 return defer.succeed(self._object_id) 828 829 # failing that, get it from the DB; multiple calls to this function 830 # at the same time will not hurt 831 try: 832 hostname = os.uname()[1] # only on unix 833 except AttributeError: 834 hostname = socket.getfqdn() 835 master_name = "%s:%s" % (hostname, os.path.abspath(self.basedir)) 836 837 d = self.db.state.getObjectId(master_name, "BuildMaster") 838 def keep(id): 839 self._object_id = id

840 d.addCallback(keep) 841 return d 842 843 844 ## triggering methods and subscriptions 845

846 - def addChange(self, who=None, files=None, comments=None, author=None, 847 isdir=None, is_dir=None, links=None, revision=None, when=None, 848 when_timestamp=None, branch=None, category=None, revlink='', 849 properties={}, repository='', project=''):

850 """ 851 Add a change to the buildmaster and act on it. 852 853 This is a wrapper around L{ChangesConnectorComponent.addChange} which 854 also acts on the resulting change and returns a L{Change} instance. 855 856 Note that all parameters are keyword arguments, although C{who}, 857 C{files}, and C{comments} can be specified positionally for 858 backward-compatibility. 859 860 @param author: the author of this change 861 @type author: unicode string 862 863 @param who: deprecated name for C{author} 864 865 @param files: a list of filenames that were changed 866 @type branch: list of unicode strings 867 868 @param comments: user comments on the change 869 @type branch: unicode string 870 871 @param is_dir: deprecated 872 873 @param isdir: deprecated name for C{is_dir} 874 875 @param links: a list of links related to this change, e.g., to web 876 viewers or review pages 877 @type links: list of unicode strings 878 879 @param revision: the revision identifier for this change 880 @type revision: unicode string 881 882 @param when_timestamp: when this change occurred, or the current time 883 if None 884 @type when_timestamp: datetime instance or None 885 886 @param when: deprecated name and type for C{when_timestamp} 887 @type when: integer (UNIX epoch time) or None 888 889 @param branch: the branch on which this change took place 890 @type branch: unicode string 891 892 @param category: category for this change (arbitrary use by Buildbot 893 users) 894 @type category: unicode string 895 896 @param revlink: link to a web view of this revision 897 @type revlink: unicode string 898 899 @param properties: properties to set on this change 900 @type properties: dictionary with string keys and simple values 901 (JSON-able). Note that the property source is I{not} included 902 in this dictionary. 903 904 @param repository: the repository in which this change took place 905 @type repository: unicode string 906 907 @param project: the project this change is a part of 908 @type project: unicode string 909 910 @returns: L{Change} instance via Deferred 911 """ 912 metrics.MetricCountEvent.log("added_changes", 1) 913 914 # handle translating deprecated names into new names for db.changes 915 def handle_deprec(oldname, old, newname, new, default=None, 916 converter = lambda x:x): 917 if old is not None: 918 if new is None: 919 log.msg("WARNING: change source is using deprecated " 920 "addChange parameter '%s'" % oldname) 921 return converter(old) 922 raise TypeError("Cannot provide '%s' and '%s' to addChange" 923 % (oldname, newname)) 924 if new is None: 925 new = default 926 return new

927 928 author = handle_deprec("who", who, "author", author) 929 is_dir = handle_deprec("isdir", isdir, "is_dir", is_dir, 930 default=0) 931 when_timestamp = handle_deprec("when", when, 932 "when_timestamp", when_timestamp, 933 converter=epoch2datetime) 934 935 # add a source to each property 936 for n in properties: 937 properties[n] = (properties[n], 'Change') 938 939 d = self.db.changes.addChange(author=author, files=files, 940 comments=comments, is_dir=is_dir, links=links, 941 revision=revision, when_timestamp=when_timestamp, 942 branch=branch, category=category, revlink=revlink, 943 properties=properties, repository=repository, project=project) 944 945 # convert the changeid to a Change instance 946 d.addCallback(lambda changeid : 947 self.db.changes.getChange(changeid)) 948 d.addCallback(lambda chdict : 949 changes.Change.fromChdict(self, chdict)) 950 951 def notify(change): 952 msg = u"added change %s to database" % change 953 log.msg(msg.encode('utf-8', 'replace')) 954 # only deliver messages immediately if we're not polling 955 if not self.db_poll_interval: 956 self._change_subs.deliver(change) 957 return change 958 d.addCallback(notify) 959 return d 960

961 - def subscribeToChanges(self, callback):

962 """ 963 Request that C{callback} be called with each Change object added to the 964 cluster. 965 966 Note: this method will go away in 0.9.x 967 """ 968 return self._change_subs.subscribe(callback)

969

970 - def addBuildset(self, **kwargs):

971 """ 972 Add a buildset to the buildmaster and act on it. Interface is 973 identical to 974 L{buildbot.db.buildsets.BuildsetConnectorComponent.addBuildset}, 975 including returning a Deferred, but also potentially triggers the 976 resulting builds. 977 """ 978 d = self.db.buildsets.addBuildset(**kwargs) 979 def notify((bsid,brids)): 980 log.msg("added buildset %d to database" % bsid) 981 # note that buildset additions are only reported on this master 982 self._new_buildset_subs.deliver(bsid=bsid, **kwargs) 983 # only deliver messages immediately if we're not polling 984 if not self.db_poll_interval: 985 for bn, brid in brids.iteritems(): 986 self.buildRequestAdded(bsid=bsid, brid=brid, 987 buildername=bn) 988 return (bsid,brids)

989 d.addCallback(notify) 990 return d 991

992 - def subscribeToBuildsets(self, callback):

993 """ 994 Request that C{callback(bsid=bsid, ssid=ssid, reason=reason, 995 properties=properties, builderNames=builderNames, 996 external_idstring=external_idstring)} be called whenever a buildset is 997 added. Properties is a dictionary as expected for 998 L{BuildsetsConnectorComponent.addBuildset}. 999 1000 Note that this only works for buildsets added on this master. 1001 1002 Note: this method will go away in 0.9.x 1003 """ 1004 return self._new_buildset_subs.subscribe(callback)

1005 1006 @defer.deferredGenerator

1007 - def maybeBuildsetComplete(self, bsid):

1008 """ 1009 Instructs the master to check whether the buildset is complete, 1010 and notify appropriately if it is. 1011 1012 Note that buildset completions are only reported on the master 1013 on which the last build request completes. 1014 """ 1015 wfd = defer.waitForDeferred( 1016 self.db.buildrequests.getBuildRequests(bsid=bsid, complete=False)) 1017 yield wfd 1018 brdicts = wfd.getResult() 1019 1020 # if there are incomplete buildrequests, bail out 1021 if brdicts: 1022 return 1023 1024 wfd = defer.waitForDeferred( 1025 self.db.buildrequests.getBuildRequests(bsid=bsid)) 1026 yield wfd 1027 brdicts = wfd.getResult() 1028 1029 # figure out the overall results of the buildset 1030 cumulative_results = SUCCESS 1031 for brdict in brdicts: 1032 if brdict['results'] not in (SUCCESS, WARNINGS): 1033 cumulative_results = FAILURE 1034 1035 # mark it as completed in the database 1036 wfd = defer.waitForDeferred( 1037 self.db.buildsets.completeBuildset(bsid, cumulative_results)) 1038 yield wfd 1039 wfd.getResult() 1040 1041 # and deliver to any listeners 1042 self._buildsetComplete(bsid, cumulative_results)

1043

1044 - def _buildsetComplete(self, bsid, results):

1045 self._complete_buildset_subs.deliver(bsid, results)

1046

1047 - def subscribeToBuildsetCompletions(self, callback):

1048 """ 1049 Request that C{callback(bsid, result)} be called whenever a 1050 buildset is complete. 1051 1052 Note: this method will go away in 0.9.x 1053 """ 1054 return self._complete_buildset_subs.subscribe(callback)

1055

1056 - def buildRequestAdded(self, bsid, brid, buildername):

1057 """ 1058 Notifies the master that a build request is available to be claimed; 1059 this may be a brand new build request, or a build request that was 1060 previously claimed and unclaimed through a timeout or other calamity. 1061 1062 @param bsid: containing buildset id 1063 @param brid: buildrequest ID 1064 @param buildername: builder named by the build request 1065 """ 1066 self._new_buildrequest_subs.deliver( 1067 dict(bsid=bsid, brid=brid, buildername=buildername))

1068

1069 - def subscribeToBuildRequests(self, callback):

1070 """ 1071 Request that C{callback} be invoked with a dictionary with keys C{brid} 1072 (the build request id), C{bsid} (buildset id) and C{buildername} 1073 whenever a new build request is added to the database. Note that, due 1074 to the delayed nature of subscriptions, the build request may already 1075 be claimed by the time C{callback} is invoked. 1076 1077 Note: this method will go away in 0.9.x 1078 """ 1079 return self._new_buildrequest_subs.subscribe(callback)

1080 1081 1082 ## database polling 1083

1084 - def pollDatabase(self):

1085 # poll each of the tables that can indicate new, actionable stuff for 1086 # this buildmaster to do. This is used in a TimerService, so returning 1087 # a Deferred means that we won't run two polling operations 1088 # simultaneously. Each particular poll method handles errors itself, 1089 # although catastrophic errors are handled here 1090 d = defer.gatherResults([ 1091 self.pollDatabaseChanges(), 1092 self.pollDatabaseBuildRequests(), 1093 # also unclaim 1094 ]) 1095 d.addErrback(log.err, 'while polling database') 1096 return d

1097 1098 _last_processed_change = None 1099 @defer.deferredGenerator

1100 - def pollDatabaseChanges(self):

1101 # Older versions of Buildbot had each scheduler polling the database 1102 # independently, and storing a "last_processed" state indicating the 1103 # last change it had processed. This had the advantage of allowing 1104 # schedulers to pick up changes that arrived in the database while 1105 # the scheduler was not running, but was horribly inefficient. 1106 1107 # This version polls the database on behalf of the schedulers, using a 1108 # similar state at the master level. 1109 1110 timer = metrics.Timer("BuildMaster.pollDatabaseChanges()") 1111 timer.start() 1112 1113 need_setState = False 1114 1115 # get the last processed change id 1116 if self._last_processed_change is None: 1117 wfd = defer.waitForDeferred( 1118 self._getState('last_processed_change')) 1119 yield wfd 1120 self._last_processed_change = wfd.getResult() 1121 1122 # if it's still None, assume we've processed up to the latest changeid 1123 if self._last_processed_change is None: 1124 wfd = defer.waitForDeferred( 1125 self.db.changes.getLatestChangeid()) 1126 yield wfd 1127 lpc = wfd.getResult() 1128 # if there *are* no changes, count the last as '0' so that we don't 1129 # skip the first change 1130 if lpc is None: 1131 lpc = 0 1132 self._last_processed_change = lpc 1133 1134 need_setState = True 1135 1136 if self._last_processed_change is None: 1137 timer.stop() 1138 return 1139 1140 while True: 1141 changeid = self._last_processed_change + 1 1142 wfd = defer.waitForDeferred( 1143 self.db.changes.getChange(changeid)) 1144 yield wfd 1145 chdict = wfd.getResult() 1146 1147 # if there's no such change, we've reached the end and can 1148 # stop polling 1149 if not chdict: 1150 break 1151 1152 wfd = defer.waitForDeferred( 1153 changes.Change.fromChdict(self, chdict)) 1154 yield wfd 1155 change = wfd.getResult() 1156 1157 self._change_subs.deliver(change) 1158 1159 self._last_processed_change = changeid 1160 need_setState = True 1161 1162 # write back the updated state, if it's changed 1163 if need_setState: 1164 wfd = defer.waitForDeferred( 1165 self._setState('last_processed_change', 1166 self._last_processed_change)) 1167 yield wfd 1168 wfd.getResult() 1169 timer.stop()

1170 1171 _last_unclaimed_brids_set = None 1172 _last_claim_cleanup = 0 1173 @defer.deferredGenerator

1174 - def pollDatabaseBuildRequests(self):

1175 # deal with cleaning up unclaimed requests, and (if necessary) 1176 # requests from a previous instance of this master 1177 timer = metrics.Timer("BuildMaster.pollDatabaseBuildRequests()") 1178 timer.start() 1179 1180 # cleanup unclaimed builds 1181 since_last_cleanup = reactor.seconds() - self._last_claim_cleanup 1182 if since_last_cleanup < self.RECLAIM_BUILD_INTERVAL: 1183 unclaimed_age = (self.RECLAIM_BUILD_INTERVAL 1184 * self.UNCLAIMED_BUILD_FACTOR) 1185 wfd = defer.waitForDeferred( 1186 self.db.buildrequests.unclaimExpiredRequests(unclaimed_age)) 1187 yield wfd 1188 wfd.getResult() 1189 1190 self._last_claim_cleanup = reactor.seconds() 1191 1192 # _last_unclaimed_brids_set tracks the state of unclaimed build 1193 # requests; whenever it sees a build request which was not claimed on 1194 # the last poll, it notifies the subscribers. It only tracks that 1195 # state within the master instance, though; on startup, it notifies for 1196 # all unclaimed requests in the database. 1197 1198 last_unclaimed = self._last_unclaimed_brids_set or set() 1199 if len(last_unclaimed) > self.WARNING_UNCLAIMED_COUNT: 1200 log.msg("WARNING: %d unclaimed buildrequests - is a scheduler " 1201 "producing builds for which no builder is running?" 1202 % len(last_unclaimed)) 1203 1204 # get the current set of unclaimed buildrequests 1205 wfd = defer.waitForDeferred( 1206 self.db.buildrequests.getBuildRequests(claimed=False)) 1207 yield wfd 1208 now_unclaimed_brdicts = wfd.getResult() 1209 now_unclaimed = set([ brd['brid'] for brd in now_unclaimed_brdicts ]) 1210 1211 # and store that for next time 1212 self._last_unclaimed_brids_set = now_unclaimed 1213 1214 # see what's new, and notify if anything is 1215 new_unclaimed = now_unclaimed - last_unclaimed 1216 if new_unclaimed: 1217 brdicts = dict((brd['brid'], brd) for brd in now_unclaimed_brdicts) 1218 for brid in new_unclaimed: 1219 brd = brdicts[brid] 1220 self.buildRequestAdded(brd['buildsetid'], brd['brid'], 1221 brd['buildername']) 1222 timer.stop()

1223 1224 ## state maintenance (private) 1225 1226 _master_objectid = None 1227

1228 - def _getObjectId(self):

1229 if self._master_objectid is None: 1230 d = self.db.state.getObjectId('master', 1231 'buildbot.master.BuildMaster') 1232 def keep(objectid): 1233 self._master_objectid = objectid 1234 return objectid

1235 d.addCallback(keep) 1236 return d 1237 return defer.succeed(self._master_objectid) 1238

1239 - def _getState(self, name, default=None):

1240 "private wrapper around C{self.db.state.getState}" 1241 d = self._getObjectId() 1242 def get(objectid): 1243 return self.db.state.getState(self._master_objectid, name, default)

1244 d.addCallback(get) 1245 return d 1246

1247 - def _setState(self, name, value):

1248 "private wrapper around C{self.db.state.setState}" 1249 d = self._getObjectId() 1250 def set(objectid): 1251 return self.db.state.setState(self._master_objectid, name, value)

1252 d.addCallback(set) 1253 return d 1254

1255 -class Control:

1256 implements(interfaces.IControl) 1257

1258 - def __init__(self, master):

1259 self.master = master

1260

1261 - def addChange(self, change):

1262 self.master.addChange(change)

1263

1264 - def addBuildset(self, **kwargs):

1265 return self.master.addBuildset(**kwargs)

1266

1267 - def getBuilder(self, name):

1268 b = self.master.botmaster.builders[name] 1269 return BuilderControl(b, self)

1270 1271 components.registerAdapter(Control, BuildMaster, interfaces.IControl) 1272

Source Code for Module buildbot.master