buildbot.config

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 from __future__ import with_statement 17 18 import os 19 import re 20 import sys 21 import warnings 22 from buildbot.util import safeTranslate 23 from buildbot import interfaces 24 from buildbot import locks 25 from buildbot.revlinks import default_revlink_matcher 26 from twisted.python import log, failure 27 from twisted.internet import defer 28 from twisted.application import service

29 30 -class ConfigErrors(Exception):

31

32 - def __init__(self, errors=[]):

33 self.errors = errors[:]

34

35 - def __str__(self):

36 return "\n".join(self.errors)

37

38 - def addError(self, msg):

39 self.errors.append(msg)

40

41 - def __nonzero__(self):

42 return len(self.errors)

43 44 _errors = None

45 -def error(error):

46 if _errors is not None: 47 _errors.addError(error) 48 else: 49 raise ConfigErrors([error])

50

51 -class MasterConfig(object):

52

53 - def __init__(self):

54 # local import to avoid circular imports 55 from buildbot.process import properties 56 # default values for all attributes 57 58 # global 59 self.title = 'Buildbot' 60 self.titleURL = 'http://buildbot.net' 61 self.buildbotURL = 'http://localhost:8080/' 62 self.changeHorizon = None 63 self.eventHorizon = 50 64 self.logHorizon = None 65 self.buildHorizon = None 66 self.logCompressionLimit = 4*1024 67 self.logCompressionMethod = 'bz2' 68 self.logMaxTailSize = None 69 self.logMaxSize = None 70 self.properties = properties.Properties() 71 self.mergeRequests = None 72 self.codebaseGenerator = None 73 self.prioritizeBuilders = None 74 self.slavePortnum = None 75 self.multiMaster = False 76 self.debugPassword = None 77 self.manhole = None 78 79 self.validation = dict( 80 branch=re.compile(r'^[\w.+/~-]*$'), 81 revision=re.compile(r'^[ \w\.\-\/]*$'), 82 property_name=re.compile(r'^[\w\.\-\/\~:]*$'), 83 property_value=re.compile(r'^[\w\.\-\/\~:]*$'), 84 ) 85 self.db = dict( 86 db_url='sqlite:///state.sqlite', 87 db_poll_interval=None, 88 ) 89 self.metrics = None 90 self.caches = dict( 91 Builds=15, 92 Changes=10, 93 ) 94 self.schedulers = {} 95 self.builders = [] 96 self.slaves = [] 97 self.change_sources = [] 98 self.status = [] 99 self.user_managers = [] 100 self.revlink = default_revlink_matcher

101 102 _known_config_keys = set([ 103 "buildbotURL", "buildCacheSize", "builders", "buildHorizon", "caches", 104 "change_source", "codebaseGenerator", "changeCacheSize", "changeHorizon", 105 'db', "db_poll_interval", "db_url", "debugPassword", "eventHorizon", 106 "logCompressionLimit", "logCompressionMethod", "logHorizon", 107 "logMaxSize", "logMaxTailSize", "manhole", "mergeRequests", "metrics", 108 "multiMaster", "prioritizeBuilders", "projectName", "projectURL", 109 "properties", "revlink", "schedulers", "slavePortnum", "slaves", 110 "status", "title", "titleURL", "user_managers", "validation" 111 ]) 112 113 @classmethod

114 - def loadConfig(cls, basedir, filename):

115 if not os.path.isdir(basedir): 116 raise ConfigErrors([ 117 "basedir '%s' does not exist" % (basedir,), 118 ]) 119 filename = os.path.join(basedir, filename) 120 if not os.path.exists(filename): 121 raise ConfigErrors([ 122 "configuration file '%s' does not exist" % (filename,), 123 ]) 124 125 try: 126 f = open(filename, "r") 127 except IOError, e: 128 raise ConfigErrors([ 129 "unable to open configuration file %r: %s" % (filename, e), 130 ]) 131 132 log.msg("Loading configuration from %r" % (filename,)) 133 134 # execute the config file 135 localDict = { 136 'basedir': os.path.expanduser(basedir), 137 '__file__': os.path.abspath(filename), 138 } 139 140 # from here on out we can batch errors together for the user's 141 # convenience 142 global _errors 143 _errors = errors = ConfigErrors() 144 145 old_sys_path = sys.path[:] 146 sys.path.append(basedir) 147 try: 148 try: 149 exec f in localDict 150 except ConfigErrors, e: 151 for error in e.errors: 152 errors.addError(error) 153 raise errors 154 except: 155 log.err(failure.Failure(), 'error while parsing config file:') 156 errors.addError( 157 "error while parsing config file: %s (traceback in logfile)" % 158 (sys.exc_info()[1],), 159 ) 160 raise errors 161 finally: 162 f.close() 163 sys.path[:] = old_sys_path 164 _errors = None 165 166 if 'BuildmasterConfig' not in localDict: 167 errors.addError( 168 "Configuration file %r does not define 'BuildmasterConfig'" 169 % (filename,), 170 ) 171 raise errors 172 173 config_dict = localDict['BuildmasterConfig'] 174 175 # check for unknown keys 176 unknown_keys = set(config_dict.keys()) - cls._known_config_keys 177 if unknown_keys: 178 if len(unknown_keys) == 1: 179 errors.addError('Unknown BuildmasterConfig key %s' % 180 (unknown_keys.pop())) 181 else: 182 errors.addError('Unknown BuildmasterConfig keys %s' % 183 (', '.join(sorted(unknown_keys)))) 184 185 # instantiate a new config object, which will apply defaults 186 # automatically 187 config = cls() 188 189 # and defer the rest to sub-functions, for code clarity 190 config.load_global(filename, config_dict, errors) 191 config.load_validation(filename, config_dict, errors) 192 config.load_db(filename, config_dict, errors) 193 config.load_metrics(filename, config_dict, errors) 194 config.load_caches(filename, config_dict, errors) 195 config.load_schedulers(filename, config_dict, errors) 196 config.load_builders(filename, config_dict, errors) 197 config.load_slaves(filename, config_dict, errors) 198 config.load_change_sources(filename, config_dict, errors) 199 config.load_status(filename, config_dict, errors) 200 config.load_user_managers(filename, config_dict, errors) 201 202 # run some sanity checks 203 config.check_single_master(errors) 204 config.check_schedulers(errors) 205 config.check_locks(errors) 206 config.check_builders(errors) 207 config.check_status(errors) 208 config.check_horizons(errors) 209 config.check_slavePortnum(errors) 210 211 if errors: 212 raise errors 213 214 return config

215

216 - def load_global(self, filename, config_dict, errors):

217 def copy_param(name, alt_key=None, 218 check_type=None, check_type_name=None): 219 if name in config_dict: 220 v = config_dict[name] 221 elif alt_key and alt_key in config_dict: 222 v = config_dict[alt_key] 223 else: 224 return 225 if v is not None and check_type and not isinstance(v, check_type): 226 errors.addError("c['%s'] must be %s" % 227 (name, check_type_name)) 228 else: 229 setattr(self, name, v)

230 231 def copy_int_param(name, alt_key=None): 232 copy_param(name, alt_key=alt_key, 233 check_type=int, check_type_name='an int')

234 235 def copy_str_param(name, alt_key=None): 236 copy_param(name, alt_key=alt_key, 237 check_type=basestring, check_type_name='a string') 238 239 copy_str_param('title', alt_key='projectName') 240 copy_str_param('titleURL', alt_key='projectURL') 241 copy_str_param('buildbotURL') 242 243 copy_int_param('changeHorizon') 244 copy_int_param('eventHorizon') 245 copy_int_param('logHorizon') 246 copy_int_param('buildHorizon') 247 248 copy_int_param('logCompressionLimit') 249 250 if 'logCompressionMethod' in config_dict: 251 logCompressionMethod = config_dict.get('logCompressionMethod') 252 if logCompressionMethod not in ('bz2', 'gz'): 253 errors.addError( 254 "c['logCompressionMethod'] must be 'bz2' or 'gz'") 255 self.logCompressionMethod = logCompressionMethod 256 257 copy_int_param('logMaxSize') 258 copy_int_param('logMaxTailSize') 259 260 properties = config_dict.get('properties', {}) 261 if not isinstance(properties, dict): 262 errors.addError("c['properties'] must be a dictionary") 263 else: 264 self.properties.update(properties, filename) 265 266 mergeRequests = config_dict.get('mergeRequests') 267 if (mergeRequests not in (None, True, False) 268 and not callable(mergeRequests)): 269 errors.addError("mergeRequests must be a callable, True, or False") 270 else: 271 self.mergeRequests = mergeRequests 272 273 codebaseGenerator = config_dict.get('codebaseGenerator') 274 if (codebaseGenerator is not None and 275 not callable(codebaseGenerator)): 276 errors.addError("codebaseGenerator must be a callable accepting a dict and returning a str") 277 else: 278 self.codebaseGenerator = codebaseGenerator 279 280 prioritizeBuilders = config_dict.get('prioritizeBuilders') 281 if prioritizeBuilders is not None and not callable(prioritizeBuilders): 282 errors.addError("prioritizeBuilders must be a callable") 283 else: 284 self.prioritizeBuilders = prioritizeBuilders 285 286 if 'slavePortnum' in config_dict: 287 slavePortnum = config_dict.get('slavePortnum') 288 if isinstance(slavePortnum, int): 289 slavePortnum = "tcp:%d" % slavePortnum 290 self.slavePortnum = slavePortnum 291 292 if 'multiMaster' in config_dict: 293 self.multiMaster = config_dict["multiMaster"] 294 295 copy_str_param('debugPassword') 296 297 if 'manhole' in config_dict: 298 # we don't check that this is a manhole instance, since that 299 # requires importing buildbot.manhole for every user, and currently 300 # that will fail if pycrypto isn't installed 301 self.manhole = config_dict['manhole'] 302 303 if 'revlink' in config_dict: 304 revlink = config_dict['revlink'] 305 if not callable(revlink): 306 errors.addError("revlink must be a callable") 307 else: 308 self.revlink = revlink 309

310 - def load_validation(self, filename, config_dict, errors):

311 validation = config_dict.get("validation", {}) 312 if not isinstance(validation, dict): 313 errors.addError("c['validation'] must be a dictionary") 314 else: 315 unknown_keys = ( 316 set(validation.keys()) - set(self.validation.keys())) 317 if unknown_keys: 318 errors.addError("unrecognized validation key(s): %s" % 319 (", ".join(unknown_keys))) 320 else: 321 self.validation.update(validation)

322 323

324 - def load_db(self, filename, config_dict, errors):

325 if 'db' in config_dict: 326 db = config_dict['db'] 327 if set(db.keys()) > set(['db_url', 'db_poll_interval']): 328 errors.addError("unrecognized keys in c['db']") 329 self.db.update(db) 330 if 'db_url' in config_dict: 331 self.db['db_url'] = config_dict['db_url'] 332 if 'db_poll_interval' in config_dict: 333 self.db['db_poll_interval'] = config_dict["db_poll_interval"] 334 335 # we don't attempt to parse db URLs here - the engine strategy will do so 336 337 # check the db_poll_interval 338 db_poll_interval = self.db['db_poll_interval'] 339 if db_poll_interval is not None and \ 340 not isinstance(db_poll_interval, int): 341 errors.addError("c['db_poll_interval'] must be an int") 342 else: 343 self.db['db_poll_interval'] = db_poll_interval

344 345

346 - def load_metrics(self, filename, config_dict, errors):

347 # we don't try to validate metrics keys 348 if 'metrics' in config_dict: 349 metrics = config_dict["metrics"] 350 if not isinstance(metrics, dict): 351 errors.addError("c['metrics'] must be a dictionary") 352 else: 353 self.metrics = metrics

354 355

356 - def load_caches(self, filename, config_dict, errors):

357 explicit = False 358 if 'caches' in config_dict: 359 explicit = True 360 caches = config_dict['caches'] 361 if not isinstance(caches, dict): 362 errors.addError("c['caches'] must be a dictionary") 363 else: 364 valPairs = caches.items() 365 for (x, y) in valPairs: 366 if (not isinstance(y, int)): 367 errors.addError( 368 "value for cache size '%s' must be an integer" % x) 369 self.caches.update(caches) 370 371 if 'buildCacheSize' in config_dict: 372 if explicit: 373 msg = "cannot specify c['caches'] and c['buildCacheSize']" 374 errors.addError(msg) 375 self.caches['Builds'] = config_dict['buildCacheSize'] 376 if 'changeCacheSize' in config_dict: 377 if explicit: 378 msg = "cannot specify c['caches'] and c['changeCacheSize']" 379 errors.addError(msg) 380 self.caches['Changes'] = config_dict['changeCacheSize']

381 382

383 - def load_schedulers(self, filename, config_dict, errors):

384 if 'schedulers' not in config_dict: 385 return 386 schedulers = config_dict['schedulers'] 387 388 ok = True 389 if not isinstance(schedulers, (list, tuple)): 390 ok = False 391 else: 392 for s in schedulers: 393 if not interfaces.IScheduler.providedBy(s): 394 ok = False 395 if not ok: 396 msg="c['schedulers'] must be a list of Scheduler instances" 397 errors.addError(msg) 398 399 # convert from list to dict, first looking for duplicates 400 seen_names = set() 401 for s in schedulers: 402 if s.name in seen_names: 403 errors.addError("scheduler name '%s' used multiple times" % 404 s.name) 405 seen_names.add(s.name) 406 407 self.schedulers = dict((s.name, s) for s in schedulers)

408 409

410 - def load_builders(self, filename, config_dict, errors):

411 if 'builders' not in config_dict: 412 return 413 builders = config_dict['builders'] 414 415 if not isinstance(builders, (list, tuple)): 416 errors.addError("c['builders'] must be a list") 417 return 418 419 # convert all builder configs to BuilderConfig instances 420 def mapper(b): 421 if isinstance(b, BuilderConfig): 422 return b 423 elif isinstance(b, dict): 424 return BuilderConfig(**b) 425 else: 426 raise RuntimeError() # signal for the try/except below

427 try: 428 builders = [ mapper(b) for b in builders ] 429 except RuntimeError: 430 errors.addError("c['builders'] must be a list of builder configs") 431 return 432 433 for builder in builders: 434 if os.path.isabs(builder.builddir): 435 warnings.warn("Absolute path '%s' for builder may cause " 436 "mayhem. Perhaps you meant to specify slavebuilddir " 437 "instead.") 438 439 self.builders = builders 440 441

442 - def load_slaves(self, filename, config_dict, errors):

443 if 'slaves' not in config_dict: 444 return 445 slaves = config_dict['slaves'] 446 447 if not isinstance(slaves, (list, tuple)): 448 errors.addError("c['slaves'] must be a list") 449 return 450 451 for sl in slaves: 452 if not interfaces.IBuildSlave.providedBy(sl): 453 msg = "c['slaves'] must be a list of BuildSlave instances" 454 errors.addError(msg) 455 return 456 457 if sl.slavename in ("debug", "change", "status"): 458 msg = "slave name '%s' is reserved" % sl.slavename 459 errors.addError(msg) 460 461 self.slaves = config_dict['slaves']

462 463

464 - def load_change_sources(self, filename, config_dict, errors):

465 change_source = config_dict.get('change_source', []) 466 if isinstance(change_source, (list, tuple)): 467 change_sources = change_source 468 else: 469 change_sources = [change_source] 470 471 for s in change_sources: 472 if not interfaces.IChangeSource.providedBy(s): 473 msg = "c['change_source'] must be a list of change sources" 474 errors.addError(msg) 475 return 476 477 self.change_sources = change_sources

478

479 - def load_status(self, filename, config_dict, errors):

480 if 'status' not in config_dict: 481 return 482 status = config_dict.get('status', []) 483 484 msg = "c['status'] must be a list of status receivers" 485 if not isinstance(status, (list, tuple)): 486 errors.addError(msg) 487 return 488 489 for s in status: 490 if not interfaces.IStatusReceiver.providedBy(s): 491 errors.addError(msg) 492 return 493 494 self.status = status

495 496

497 - def load_user_managers(self, filename, config_dict, errors):

498 if 'user_managers' not in config_dict: 499 return 500 user_managers = config_dict['user_managers'] 501 502 msg = "c['user_managers'] must be a list of user managers" 503 if not isinstance(user_managers, (list, tuple)): 504 errors.addError(msg) 505 return 506 507 self.user_managers = user_managers

508 509

510 - def check_single_master(self, errors):

511 # check additional problems that are only valid in a single-master 512 # installation 513 if self.multiMaster: 514 return 515 516 if not self.slaves: 517 errors.addError("no slaves are configured") 518 519 if not self.builders: 520 errors.addError("no builders are configured") 521 522 # check that all builders are implemented on this master 523 unscheduled_buildernames = set([ b.name for b in self.builders ]) 524 for s in self.schedulers.itervalues(): 525 for n in s.listBuilderNames(): 526 if n in unscheduled_buildernames: 527 unscheduled_buildernames.remove(n) 528 if unscheduled_buildernames: 529 errors.addError("builder(s) %s have no schedulers to drive them" 530 % (', '.join(unscheduled_buildernames),))

531 532

533 - def check_schedulers(self, errors):

534 all_buildernames = set([ b.name for b in self.builders ]) 535 536 for s in self.schedulers.itervalues(): 537 for n in s.listBuilderNames(): 538 if n not in all_buildernames: 539 errors.addError("Unknown builder '%s' in scheduler '%s'" 540 % (n, s.name))

541 542

543 - def check_locks(self, errors):

544 # assert that all locks used by the Builds and their Steps are 545 # uniquely named. 546 lock_dict = {} 547 def check_lock(l): 548 if isinstance(l, locks.LockAccess): 549 l = l.lockid 550 if lock_dict.has_key(l.name): 551 if lock_dict[l.name] is not l: 552 msg = "Two locks share the same name, '%s'" % l.name 553 errors.addError(msg) 554 else: 555 lock_dict[l.name] = l

556 557 for b in self.builders: 558 if b.locks: 559 for l in b.locks: 560 check_lock(l) 561

562 - def check_builders(self, errors):

563 # look both for duplicate builder names, and for builders pointing 564 # to unknown slaves 565 slavenames = set([ s.slavename for s in self.slaves ]) 566 seen_names = set() 567 seen_builddirs = set() 568 569 for b in self.builders: 570 unknowns = set(b.slavenames) - slavenames 571 if unknowns: 572 errors.addError("builder '%s' uses unknown slaves %s" % 573 (b.name, ", ".join(`u` for u in unknowns))) 574 if b.name in seen_names: 575 errors.addError("duplicate builder name '%s'" % b.name) 576 seen_names.add(b.name) 577 578 if b.builddir in seen_builddirs: 579 errors.addError("duplicate builder builddir '%s'" % b.builddir) 580 seen_builddirs.add(b.builddir)

581 582

583 - def check_status(self, errors):

584 # allow status receivers to check themselves against the rest of the 585 # receivers 586 for s in self.status: 587 s.checkConfig(self.status, errors)

588 589

590 - def check_horizons(self, errors):

591 if self.logHorizon is not None and self.buildHorizon is not None: 592 if self.logHorizon > self.buildHorizon: 593 errors.addError( 594 "logHorizon must be less than or equal to buildHorizon")

595

596 - def check_slavePortnum(self, errors):

597 if self.slavePortnum: 598 return 599 600 if self.slaves: 601 errors.addError( 602 "slaves are configured, but no slavePortnum is set") 603 if self.debugPassword: 604 errors.addError( 605 "debug client is configured, but no slavePortnum is set")

606

607 608 -class BuilderConfig:

609

610 - def __init__(self, name=None, slavename=None, slavenames=None, 611 builddir=None, slavebuilddir=None, factory=None, category=None, 612 nextSlave=None, nextBuild=None, locks=None, env=None, 613 properties=None, mergeRequests=None):

614 615 errors = ConfigErrors([]) 616 617 # name is required, and can't start with '_' 618 if not name or type(name) not in (str, unicode): 619 errors.addError("builder's name is required") 620 name = '<unknown>' 621 elif name[0] == '_': 622 errors.addError( 623 "builder names must not start with an underscore: '%s'" % name) 624 self.name = name 625 626 # factory is required 627 if factory is None: 628 errors.addError("builder '%s' has no factory" % name) 629 from buildbot.process.factory import BuildFactory 630 if factory is not None and not isinstance(factory, BuildFactory): 631 errors.addError( 632 "builder '%s's factory is not a BuildFactory instance" % name) 633 self.factory = factory 634 635 # slavenames can be a single slave name or a list, and should also 636 # include slavename, if given 637 if type(slavenames) is str: 638 slavenames = [ slavenames ] 639 if slavenames: 640 if not isinstance(slavenames, list): 641 errors.addError( 642 "builder '%s': slavenames must be a list or a string" % 643 (name,)) 644 else: 645 slavenames = [] 646 647 if slavename: 648 if type(slavename) != str: 649 errors.addError( 650 "builder '%s': slavename must be a string" % (name,)) 651 slavenames = slavenames + [ slavename ] 652 if not slavenames: 653 errors.addError( 654 "builder '%s': at least one slavename is required" % (name,)) 655 656 self.slavenames = slavenames 657 658 # builddir defaults to name 659 if builddir is None: 660 builddir = safeTranslate(name) 661 self.builddir = builddir 662 663 # slavebuilddir defaults to builddir 664 if slavebuilddir is None: 665 slavebuilddir = builddir 666 self.slavebuilddir = slavebuilddir 667 668 # remainder are optional 669 if category is not None and not isinstance(category, str): 670 errors.addError( 671 "builder '%s': category must be a string" % (name,)) 672 673 self.category = category or '' 674 self.nextSlave = nextSlave 675 if nextSlave and not callable(nextSlave): 676 errors.addError('nextSlave must be a callable') 677 self.nextBuild = nextBuild 678 if nextBuild and not callable(nextBuild): 679 errors.addError('nextBuild must be a callable') 680 self.locks = locks or [] 681 self.env = env or {} 682 if not isinstance(self.env, dict): 683 errors.addError("builder's env must be a dictionary") 684 self.properties = properties or {} 685 self.mergeRequests = mergeRequests 686 687 if errors: 688 raise errors

689 690

691 - def getConfigDict(self):

692 # note: this method will disappear eventually - put your smarts in the 693 # constructor! 694 rv = { 695 'name': self.name, 696 'slavenames': self.slavenames, 697 'factory': self.factory, 698 'builddir': self.builddir, 699 'slavebuilddir': self.slavebuilddir, 700 } 701 if self.category: 702 rv['category'] = self.category 703 if self.nextSlave: 704 rv['nextSlave'] = self.nextSlave 705 if self.nextBuild: 706 rv['nextBuild'] = self.nextBuild 707 if self.locks: 708 rv['locks'] = self.locks 709 if self.env: 710 rv['env'] = self.env 711 if self.properties: 712 rv['properties'] = self.properties 713 if self.mergeRequests: 714 rv['mergeRequests'] = self.mergeRequests 715 return rv

716

717 718 -class ReconfigurableServiceMixin:

719 720 reconfig_priority = 128 721 722 @defer.inlineCallbacks

723 - def reconfigService(self, new_config):

724 if not service.IServiceCollection.providedBy(self): 725 return 726 727 # get a list of child services to reconfigure 728 reconfigurable_services = [ svc 729 for svc in self 730 if isinstance(svc, ReconfigurableServiceMixin) ] 731 732 # sort by priority 733 reconfigurable_services.sort(key=lambda svc : -svc.reconfig_priority) 734 735 for svc in reconfigurable_services: 736 yield svc.reconfigService(new_config)

737

Source Code for Module buildbot.config