buildbot.config

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 import re 17 import os 18 import sys 19 from buildbot.util import safeTranslate 20 from buildbot.process import properties 21 from buildbot import interfaces 22 from buildbot import locks 23 from buildbot.revlinks import default_revlink_matcher 24 from twisted.python import log, failure 25 from twisted.internet import defer 26 from twisted.application import service

27 28 -class ConfigErrors(Exception):

29

30 - def __init__(self, errors=[]):

31 self.errors = errors[:]

32

33 - def __str__(self):

34 return "\n".join(self.errors)

35

36 - def addError(self, msg):

37 self.errors.append(msg)

38

39 - def __nonzero__(self):

40 return len(self.errors)

41 42 _errors = None

43 -def error(error):

44 if _errors is not None: 45 _errors.addError(error) 46 else: 47 raise ConfigErrors([error])

48

49 -class MasterConfig(object):

50

51 - def __init__(self):

52 # default values for all attributes 53 54 # global 55 self.title = 'Buildbot' 56 self.titleURL = 'http://buildbot.net' 57 self.buildbotURL = 'http://localhost:8080/' 58 self.changeHorizon = None 59 self.eventHorizon = 50 60 self.logHorizon = None 61 self.buildHorizon = None 62 self.logCompressionLimit = 4*1024 63 self.logCompressionMethod = 'bz2' 64 self.logMaxTailSize = None 65 self.logMaxSize = None 66 self.properties = properties.Properties() 67 self.mergeRequests = None 68 self.prioritizeBuilders = None 69 self.slavePortnum = None 70 self.multiMaster = False 71 self.debugPassword = None 72 self.manhole = None 73 74 self.validation = dict( 75 branch=re.compile(r'^[\w.+/~-]*$'), 76 revision=re.compile(r'^[ \w\.\-\/]*$'), 77 property_name=re.compile(r'^[\w\.\-\/\~:]*$'), 78 property_value=re.compile(r'^[\w\.\-\/\~:]*$'), 79 ) 80 self.db = dict( 81 db_url='sqlite:///state.sqlite', 82 db_poll_interval=None, 83 ) 84 self.metrics = None 85 self.caches = dict( 86 Builds=15, 87 Changes=10, 88 ) 89 self.schedulers = {} 90 self.builders = [] 91 self.slaves = [] 92 self.change_sources = [] 93 self.status = [] 94 self.user_managers = [] 95 self.revlink = default_revlink_matcher

96 97 _known_config_keys = set([ 98 "buildbotURL", "buildCacheSize", "builders", "buildHorizon", "caches", 99 "change_source", "changeCacheSize", "changeHorizon", 100 'db', "db_poll_interval", "db_url", "debugPassword", "eventHorizon", 101 "logCompressionLimit", "logCompressionMethod", "logHorizon", 102 "logMaxSize", "logMaxTailSize", "manhole", "mergeRequests", "metrics", 103 "multiMaster", "prioritizeBuilders", "projectName", "projectURL", 104 "properties", "revlink", "schedulers", "slavePortnum", "slaves", 105 "status", "title", "titleURL", "user_managers", "validation" 106 ]) 107 108 @classmethod

109 - def loadConfig(cls, basedir, filename):

110 if not os.path.isdir(basedir): 111 raise ConfigErrors([ 112 "basedir '%s' does not exist" % (basedir,), 113 ]) 114 filename = os.path.join(basedir, filename) 115 if not os.path.exists(filename): 116 raise ConfigErrors([ 117 "configuration file '%s' does not exist" % (filename,), 118 ]) 119 120 try: 121 f = open(filename, "r") 122 except IOError, e: 123 raise ConfigErrors([ 124 "unable to open configuration file %r: %s" % (filename, e), 125 ]) 126 127 log.msg("Loading configuration from %r" % (filename,)) 128 129 # execute the config file 130 localDict = { 131 'basedir': os.path.expanduser(basedir), 132 '__file__': os.path.abspath(filename), 133 } 134 135 # from here on out we can batch errors together for the user's 136 # convenience 137 global _errors 138 _errors = errors = ConfigErrors() 139 140 old_sys_path = sys.path[:] 141 sys.path.append(basedir) 142 try: 143 try: 144 exec f in localDict 145 except ConfigErrors, e: 146 for error in e.errors: 147 errors.addError(error) 148 raise errors 149 except: 150 log.err(failure.Failure()) 151 errors.addError( 152 "error while parsing config file: %s (traceback in logfile)" % 153 (sys.exc_info()[1],), 154 ) 155 raise errors 156 finally: 157 sys.path[:] = old_sys_path 158 _errors = None 159 160 if 'BuildmasterConfig' not in localDict: 161 errors.addError( 162 "Configuration file %r does not define 'BuildmasterConfig'" 163 % (filename,), 164 ) 165 raise errors 166 167 config_dict = localDict['BuildmasterConfig'] 168 169 # check for unknown keys 170 unknown_keys = set(config_dict.keys()) - cls._known_config_keys 171 if unknown_keys: 172 if len(unknown_keys) == 1: 173 errors.addError('Unknown BuildmasterConfig key %s' % 174 (unknown_keys.pop())) 175 else: 176 errors.addError('Unknown BuildmasterConfig keys %s' % 177 (', '.join(sorted(unknown_keys)))) 178 179 # instantiate a new config object, which will apply defaults 180 # automatically 181 config = cls() 182 183 # and defer the rest to sub-functions, for code clarity 184 config.load_global(filename, config_dict, errors) 185 config.load_validation(filename, config_dict, errors) 186 config.load_db(filename, config_dict, errors) 187 config.load_metrics(filename, config_dict, errors) 188 config.load_caches(filename, config_dict, errors) 189 config.load_schedulers(filename, config_dict, errors) 190 config.load_builders(filename, config_dict, errors) 191 config.load_slaves(filename, config_dict, errors) 192 config.load_change_sources(filename, config_dict, errors) 193 config.load_status(filename, config_dict, errors) 194 config.load_user_managers(filename, config_dict, errors) 195 196 # run some sanity checks 197 config.check_single_master(errors) 198 config.check_schedulers(errors) 199 config.check_locks(errors) 200 config.check_builders(errors) 201 config.check_status(errors) 202 config.check_horizons(errors) 203 config.check_slavePortnum(errors) 204 205 if errors: 206 raise errors 207 208 return config

209

210 - def load_global(self, filename, config_dict, errors):

211 def copy_param(name, alt_key=None, 212 check_type=None, check_type_name=None): 213 if name in config_dict: 214 v = config_dict[name] 215 elif alt_key and alt_key in config_dict: 216 v = config_dict[alt_key] 217 else: 218 return 219 if v is not None and check_type and not isinstance(v, check_type): 220 errors.addError("c['%s'] must be %s" % 221 (name, check_type_name)) 222 else: 223 setattr(self, name, v)

224 225 def copy_int_param(name, alt_key=None): 226 copy_param(name, alt_key=alt_key, 227 check_type=int, check_type_name='an int')

228 229 def copy_str_param(name, alt_key=None): 230 copy_param(name, alt_key=alt_key, 231 check_type=basestring, check_type_name='a string') 232 233 copy_str_param('title', alt_key='projectName') 234 copy_str_param('titleURL', alt_key='projectURL') 235 copy_str_param('buildbotURL') 236 237 copy_int_param('changeHorizon') 238 copy_int_param('eventHorizon') 239 copy_int_param('logHorizon') 240 copy_int_param('buildHorizon') 241 242 copy_int_param('logCompressionLimit') 243 244 if 'logCompressionMethod' in config_dict: 245 logCompressionMethod = config_dict.get('logCompressionMethod') 246 if logCompressionMethod not in ('bz2', 'gz'): 247 errors.addError( 248 "c['logCompressionMethod'] must be 'bz2' or 'gz'") 249 self.logCompressionMethod = logCompressionMethod 250 251 copy_int_param('logMaxSize') 252 copy_int_param('logMaxTailSize') 253 254 properties = config_dict.get('properties', {}) 255 if not isinstance(properties, dict): 256 errors.addError("c['properties'] must be a dictionary") 257 else: 258 self.properties.update(properties, filename) 259 260 mergeRequests = config_dict.get('mergeRequests') 261 if (mergeRequests not in (None, True, False) 262 and not callable(mergeRequests)): 263 errors.addError("mergeRequests must be a callable, True, or False") 264 else: 265 self.mergeRequests = mergeRequests 266 267 prioritizeBuilders = config_dict.get('prioritizeBuilders') 268 if prioritizeBuilders is not None and not callable(prioritizeBuilders): 269 errors.addError("prioritizeBuilders must be a callable") 270 else: 271 self.prioritizeBuilders = prioritizeBuilders 272 273 if 'slavePortnum' in config_dict: 274 slavePortnum = config_dict.get('slavePortnum') 275 if isinstance(slavePortnum, int): 276 slavePortnum = "tcp:%d" % slavePortnum 277 self.slavePortnum = slavePortnum 278 279 if 'multiMaster' in config_dict: 280 self.multiMaster = config_dict["multiMaster"] 281 282 copy_str_param('debugPassword') 283 284 if 'manhole' in config_dict: 285 # we don't check that this is a manhole instance, since that 286 # requires importing buildbot.manhole for every user, and currently 287 # that will fail if pycrypto isn't installed 288 self.manhole = config_dict['manhole'] 289 290 if 'revlink' in config_dict: 291 revlink = config_dict['revlink'] 292 if not callable(revlink): 293 errors.addError("revlink must be a callable") 294 else: 295 self.revlink = revlink 296

297 - def load_validation(self, filename, config_dict, errors):

298 validation = config_dict.get("validation", {}) 299 if not isinstance(validation, dict): 300 errors.addError("c['validation'] must be a dictionary") 301 else: 302 unknown_keys = ( 303 set(validation.keys()) - set(self.validation.keys())) 304 if unknown_keys: 305 errors.addError("unrecognized validation key(s): %s" % 306 (", ".join(unknown_keys))) 307 else: 308 self.validation.update(validation)

309 310

311 - def load_db(self, filename, config_dict, errors):

312 if 'db' in config_dict: 313 db = config_dict['db'] 314 if set(db.keys()) > set(['db_url', 'db_poll_interval']): 315 errors.addError("unrecognized keys in c['db']") 316 self.db.update(db) 317 if 'db_url' in config_dict: 318 self.db['db_url'] = config_dict['db_url'] 319 if 'db_poll_interval' in config_dict: 320 self.db['db_poll_interval'] = config_dict["db_poll_interval"] 321 322 # we don't attempt to parse db URLs here - the engine strategy will do so 323 324 # check the db_poll_interval 325 db_poll_interval = self.db['db_poll_interval'] 326 if db_poll_interval is not None and \ 327 not isinstance(db_poll_interval, int): 328 errors.addError("c['db_poll_interval'] must be an int") 329 else: 330 self.db['db_poll_interval'] = db_poll_interval

331 332

333 - def load_metrics(self, filename, config_dict, errors):

334 # we don't try to validate metrics keys 335 if 'metrics' in config_dict: 336 metrics = config_dict["metrics"] 337 if not isinstance(metrics, dict): 338 errors.addError("c['metrics'] must be a dictionary") 339 else: 340 self.metrics = metrics

341 342

343 - def load_caches(self, filename, config_dict, errors):

344 explicit = False 345 if 'caches' in config_dict: 346 explicit = True 347 caches = config_dict['caches'] 348 if not isinstance(caches, dict): 349 errors.addError("c['caches'] must be a dictionary") 350 else: 351 self.caches.update(caches) 352 353 if 'buildCacheSize' in config_dict: 354 if explicit: 355 msg = "cannot specify c['caches'] and c['buildCacheSize']" 356 errors.addError(msg) 357 self.caches['Builds'] = config_dict['buildCacheSize'] 358 if 'changeCacheSize' in config_dict: 359 if explicit: 360 msg = "cannot specify c['caches'] and c['changeCacheSize']" 361 errors.addError(msg) 362 self.caches['Changes'] = config_dict['changeCacheSize']

363 364

365 - def load_schedulers(self, filename, config_dict, errors):

366 if 'schedulers' not in config_dict: 367 return 368 schedulers = config_dict['schedulers'] 369 370 ok = True 371 if not isinstance(schedulers, (list, tuple)): 372 ok = False 373 else: 374 for s in schedulers: 375 if not interfaces.IScheduler.providedBy(s): 376 ok = False 377 if not ok: 378 msg="c['schedulers'] must be a list of Scheduler instances" 379 errors.addError(msg) 380 381 # convert from list to dict, first looking for duplicates 382 seen_names = set() 383 for s in schedulers: 384 if s.name in seen_names: 385 errors.addError("scheduler name '%s' used multiple times" % 386 s.name) 387 seen_names.add(s.name) 388 389 self.schedulers = dict((s.name, s) for s in schedulers)

390 391

392 - def load_builders(self, filename, config_dict, errors):

393 if 'builders' not in config_dict: 394 return 395 builders = config_dict['builders'] 396 397 if not isinstance(builders, (list, tuple)): 398 errors.addError("c['builders'] must be a list") 399 return 400 401 # convert all builder configs to BuilderConfig instances 402 def mapper(b): 403 if isinstance(b, BuilderConfig): 404 return b 405 elif isinstance(b, dict): 406 return BuilderConfig(**b) 407 else: 408 raise RuntimeError() # signal for the try/except below

409 try: 410 builders = [ mapper(b) for b in builders ] 411 except RuntimeError: 412 errors.addError("c['builders'] must be a list of builder configs") 413 return 414 415 self.builders = builders 416 417

418 - def load_slaves(self, filename, config_dict, errors):

419 if 'slaves' not in config_dict: 420 return 421 slaves = config_dict['slaves'] 422 423 if not isinstance(slaves, (list, tuple)): 424 errors.addError("c['slaves'] must be a list") 425 return 426 427 for sl in slaves: 428 if not interfaces.IBuildSlave.providedBy(sl): 429 msg = "c['slaves'] must be a list of BuildSlave instances" 430 errors.addError(msg) 431 return 432 433 if sl.slavename in ("debug", "change", "status"): 434 msg = "slave name '%s' is reserved" % sl.slavename 435 errors.addError(msg) 436 437 self.slaves = config_dict['slaves']

438 439

440 - def load_change_sources(self, filename, config_dict, errors):

441 change_source = config_dict.get('change_source', []) 442 if isinstance(change_source, (list, tuple)): 443 change_sources = change_source 444 else: 445 change_sources = [change_source] 446 447 for s in change_sources: 448 if not interfaces.IChangeSource.providedBy(s): 449 msg = "c['change_source'] must be a list of change sources" 450 errors.addError(msg) 451 return 452 453 self.change_sources = change_sources

454 455

456 - def load_status(self, filename, config_dict, errors):

457 if 'status' not in config_dict: 458 return 459 status = config_dict.get('status', []) 460 461 msg = "c['status'] must be a list of status receivers" 462 if not isinstance(status, (list, tuple)): 463 errors.addError(msg) 464 return 465 466 for s in status: 467 if not interfaces.IStatusReceiver.providedBy(s): 468 errors.addError(msg) 469 return 470 471 self.status = status

472 473

474 - def load_user_managers(self, filename, config_dict, errors):

475 if 'user_managers' not in config_dict: 476 return 477 user_managers = config_dict['user_managers'] 478 479 msg = "c['user_managers'] must be a list of user managers" 480 if not isinstance(user_managers, (list, tuple)): 481 errors.addError(msg) 482 return 483 484 self.user_managers = user_managers

485 486

487 - def check_single_master(self, errors):

488 # check additional problems that are only valid in a single-master 489 # installation 490 if self.multiMaster: 491 return 492 493 if not self.slaves: 494 errors.addError("no slaves are configured") 495 496 if not self.builders: 497 errors.addError("no builders are configured") 498 499 # check that all builders are implemented on this master 500 unscheduled_buildernames = set([ b.name for b in self.builders ]) 501 for s in self.schedulers.itervalues(): 502 for n in s.listBuilderNames(): 503 if n in unscheduled_buildernames: 504 unscheduled_buildernames.remove(n) 505 if unscheduled_buildernames: 506 errors.addError("builder(s) %s have no schedulers to drive them" 507 % (', '.join(unscheduled_buildernames),))

508 509

510 - def check_schedulers(self, errors):

511 all_buildernames = set([ b.name for b in self.builders ]) 512 513 for s in self.schedulers.itervalues(): 514 for n in s.listBuilderNames(): 515 if n not in all_buildernames: 516 errors.addError("Unknown builder '%s' in scheduler '%s'" 517 % (n, s.name))

518 519

520 - def check_locks(self, errors):

521 # assert that all locks used by the Builds and their Steps are 522 # uniquely named. 523 lock_dict = {} 524 def check_lock(l): 525 if isinstance(l, locks.LockAccess): 526 l = l.lockid 527 if lock_dict.has_key(l.name): 528 if lock_dict[l.name] is not l: 529 msg = "Two locks share the same name, '%s'" % l.name 530 errors.addError(msg) 531 else: 532 lock_dict[l.name] = l

533 534 for b in self.builders: 535 if b.locks: 536 for l in b.locks: 537 check_lock(l) 538 539 # factories don't necessarily need to implement a .steps attribute 540 # but in practice most do, so we'll check that if it exists 541 if not hasattr(b.factory, 'steps'): 542 continue 543 for s in b.factory.steps: 544 for l in s[1].get('locks', []): 545 check_lock(l) 546 547

548 - def check_builders(self, errors):

549 # look both for duplicate builder names, and for builders pointing 550 # to unknown slaves 551 slavenames = set([ s.slavename for s in self.slaves ]) 552 seen_names = set() 553 seen_builddirs = set() 554 555 for b in self.builders: 556 unknowns = set(b.slavenames) - slavenames 557 if unknowns: 558 errors.addError("builder '%s' uses unknown slaves %s" % 559 (b.name, ", ".join(`u` for u in unknowns))) 560 if b.name in seen_names: 561 errors.addError("duplicate builder name '%s'" % b.name) 562 seen_names.add(b.name) 563 564 if b.builddir in seen_builddirs: 565 errors.addError("duplicate builder builddir '%s'" % b.builddir) 566 seen_builddirs.add(b.builddir)

567 568

569 - def check_status(self, errors):

570 # allow status receivers to check themselves against the rest of the 571 # receivers 572 for s in self.status: 573 s.checkConfig(self.status, errors)

574 575

576 - def check_horizons(self, errors):

577 if self.logHorizon is not None and self.buildHorizon is not None: 578 if self.logHorizon > self.buildHorizon: 579 errors.addError( 580 "logHorizon must be less than or equal to buildHorizon")

581

582 - def check_slavePortnum(self, errors):

583 if self.slavePortnum: 584 return 585 586 if self.slaves: 587 errors.addError( 588 "slaves are configured, but no slavePortnum is set") 589 if self.debugPassword: 590 errors.addError( 591 "debug client is configured, but no slavePortnum is set")

592

593 594 -class BuilderConfig:

595

596 - def __init__(self, name=None, slavename=None, slavenames=None, 597 builddir=None, slavebuilddir=None, factory=None, category=None, 598 nextSlave=None, nextBuild=None, locks=None, env=None, 599 properties=None, mergeRequests=None):

600 601 errors = ConfigErrors([]) 602 603 # name is required, and can't start with '_' 604 if not name or type(name) not in (str, unicode): 605 errors.addError("builder's name is required") 606 name = '<unknown>' 607 elif name[0] == '_': 608 errors.addError( 609 "builder names must not start with an underscore: '%s'" % name) 610 self.name = name 611 612 # factory is required 613 if factory is None: 614 errors.addError("builder '%s' has no factory" % name) 615 self.factory = factory 616 617 # slavenames can be a single slave name or a list, and should also 618 # include slavename, if given 619 if type(slavenames) is str: 620 slavenames = [ slavenames ] 621 if slavenames: 622 if not isinstance(slavenames, list): 623 errors.addError( 624 "builder '%s': slavenames must be a list or a string" % 625 (name,)) 626 else: 627 slavenames = [] 628 629 if slavename: 630 if type(slavename) != str: 631 errors.addError( 632 "builder '%s': slavename must be a string" % (name,)) 633 slavenames = slavenames + [ slavename ] 634 if not slavenames: 635 errors.addError( 636 "builder '%s': at least one slavename is required" % (name,)) 637 638 self.slavenames = slavenames 639 640 # builddir defaults to name 641 if builddir is None: 642 builddir = safeTranslate(name) 643 self.builddir = builddir 644 645 # slavebuilddir defaults to builddir 646 if slavebuilddir is None: 647 slavebuilddir = builddir 648 self.slavebuilddir = slavebuilddir 649 650 # remainder are optional 651 if category is not None and not isinstance(category, str): 652 errors.addError( 653 "builder '%s': category must be a string" % (name,)) 654 655 self.category = category or '' 656 self.nextSlave = nextSlave 657 if nextSlave and not callable(nextSlave): 658 errors.addError('nextSlave must be a callable') 659 self.nextBuild = nextBuild 660 if nextBuild and not callable(nextBuild): 661 errors.addError('nextBuild must be a callable') 662 self.locks = locks or [] 663 self.env = env or {} 664 if not isinstance(self.env, dict): 665 errors.addError("builder's env must be a dictionary") 666 self.properties = properties or {} 667 self.mergeRequests = mergeRequests 668 669 if errors: 670 raise errors

671 672

673 - def getConfigDict(self):

674 # note: this method will disappear eventually - put your smarts in the 675 # constructor! 676 rv = { 677 'name': self.name, 678 'slavenames': self.slavenames, 679 'factory': self.factory, 680 'builddir': self.builddir, 681 'slavebuilddir': self.slavebuilddir, 682 } 683 if self.category: 684 rv['category'] = self.category 685 if self.nextSlave: 686 rv['nextSlave'] = self.nextSlave 687 if self.nextBuild: 688 rv['nextBuild'] = self.nextBuild 689 if self.locks: 690 rv['locks'] = self.locks 691 if self.env: 692 rv['env'] = self.env 693 if self.properties: 694 rv['properties'] = self.properties 695 if self.mergeRequests: 696 rv['mergeRequests'] = self.mergeRequests 697 return rv

698

699 700 -class ReconfigurableServiceMixin:

701 702 reconfig_priority = 128 703 704 @defer.deferredGenerator

705 - def reconfigService(self, new_config):

706 if not service.IServiceCollection.providedBy(self): 707 return 708 709 # get a list of child services to reconfigure 710 reconfigurable_services = [ svc 711 for svc in self 712 if isinstance(svc, ReconfigurableServiceMixin) ] 713 714 # sort by priority 715 reconfigurable_services.sort(key=lambda svc : -svc.reconfig_priority) 716 717 for svc in reconfigurable_services: 718 d = svc.reconfigService(new_config) 719 wfd = defer.waitForDeferred(d) 720 yield wfd 721 wfd.getResult()

722

Source Code for Module buildbot.config