buildbot.master

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 17 import os 18 import signal 19 import socket 20 21 from zope.interface import implements 22 from twisted.python import log, components, failure 23 from twisted.internet import defer, reactor, task 24 from twisted.application import service 25 26 import buildbot 27 import buildbot.pbmanager 28 from buildbot.util import subscription, epoch2datetime 29 from buildbot.status.master import Status 30 from buildbot.changes import changes 31 from buildbot.changes.manager import ChangeManager 32 from buildbot import interfaces 33 from buildbot.process.builder import BuilderControl 34 from buildbot.db import connector 35 from buildbot.schedulers.manager import SchedulerManager 36 from buildbot.process.botmaster import BotMaster 37 from buildbot.process import debug 38 from buildbot.process import metrics 39 from buildbot.process import cache 40 from buildbot.process.users import users 41 from buildbot.process.users.manager import UserManagerManager 42 from buildbot.status.results import SUCCESS, WARNINGS, FAILURE 43 from buildbot import monkeypatches 44 from buildbot import config

45 46 ######################################## 47 48 -class LogRotation(object):

49 - def __init__(self):

50 self.rotateLength = 1 * 1000 * 1000 51 self.maxRotatedFiles = 10

52

53 -class BuildMaster(config.ReconfigurableServiceMixin, service.MultiService):

54 55 # frequency with which to reclaim running builds; this should be set to 56 # something fairly long, to avoid undue database load 57 RECLAIM_BUILD_INTERVAL = 10*60 58 59 # multiplier on RECLAIM_BUILD_INTERVAL at which a build is considered 60 # unclaimed; this should be at least 2 to avoid false positives 61 UNCLAIMED_BUILD_FACTOR = 6 62 63 # if this quantity of unclaimed build requests are present in the table, 64 # then something is probably wrong! The master will log a WARNING on every 65 # database poll operation. 66 WARNING_UNCLAIMED_COUNT = 10000 67

68 - def __init__(self, basedir, configFileName="master.cfg", umask=None):

69 service.MultiService.__init__(self) 70 self.setName("buildmaster") 71 72 self.umask = umask 73 74 self.basedir = basedir 75 assert os.path.isdir(self.basedir) 76 self.configFileName = configFileName 77 78 # set up child services 79 self.create_child_services() 80 81 # loop for polling the db 82 self.db_loop = None 83 84 # configuration / reconfiguration handling 85 self.config = config.MasterConfig() 86 self.reconfig_active = False 87 self.reconfig_requested = False 88 self.reconfig_notifier = None 89 90 # this stores parameters used in the tac file, and is accessed by the 91 # WebStatus to duplicate those values. 92 self.log_rotation = LogRotation() 93 94 # subscription points 95 self._change_subs = \ 96 subscription.SubscriptionPoint("changes") 97 self._new_buildrequest_subs = \ 98 subscription.SubscriptionPoint("buildrequest_additions") 99 self._new_buildset_subs = \ 100 subscription.SubscriptionPoint("buildset_additions") 101 self._complete_buildset_subs = \ 102 subscription.SubscriptionPoint("buildset_completion") 103 104 # local cache for this master's object ID 105 self._object_id = None

106 107

108 - def create_child_services(self):

109 # note that these are order-dependent. If you get the order wrong, 110 # you'll know it, as the master will fail to start. 111 112 self.metrics = metrics.MetricLogObserver() 113 self.metrics.setServiceParent(self) 114 115 self.caches = cache.CacheManager() 116 self.caches.setServiceParent(self) 117 118 self.pbmanager = buildbot.pbmanager.PBManager() 119 self.pbmanager.setServiceParent(self) 120 121 self.change_svc = ChangeManager(self) 122 self.change_svc.setServiceParent(self) 123 124 self.botmaster = BotMaster(self) 125 self.botmaster.setServiceParent(self) 126 127 self.scheduler_manager = SchedulerManager(self) 128 self.scheduler_manager.setServiceParent(self) 129 130 self.user_manager = UserManagerManager(self) 131 self.user_manager.setServiceParent(self) 132 133 self.db = connector.DBConnector(self, self.basedir) 134 self.db.setServiceParent(self) 135 136 self.debug = debug.DebugServices(self) 137 self.debug.setServiceParent(self) 138 139 self.status = Status(self) 140 self.status.setServiceParent(self)

141 142 # setup and reconfig handling 143 144 _already_started = False 145 @defer.inlineCallbacks

146 - def startService(self, _reactor=reactor):

147 assert not self._already_started, "can only start the master once" 148 self._already_started = True 149 150 log.msg("Starting BuildMaster -- buildbot.version: %s" % 151 buildbot.version) 152 153 # Set umask 154 if self.umask is not None: 155 os.umask(self.umask) 156 157 # first, apply all monkeypatches 158 monkeypatches.patch_all() 159 160 # we want to wait until the reactor is running, so we can call 161 # reactor.stop() for fatal errors 162 d = defer.Deferred() 163 _reactor.callWhenRunning(d.callback, None) 164 yield d 165 166 try: 167 # load the configuration file, treating errors as fatal 168 try: 169 self.config = config.MasterConfig.loadConfig(self.basedir, 170 self.configFileName) 171 except config.ConfigErrors, e: 172 log.msg("Configuration Errors:") 173 for msg in e.errors: 174 log.msg(" " + msg) 175 log.msg("Halting master.") 176 _reactor.stop() 177 return 178 except: 179 log.err(failure.Failure(), 'while starting BuildMaster') 180 _reactor.stop() 181 return 182 183 # set up services that need access to the config before everything else 184 # gets told to reconfig 185 try: 186 yield self.db.setup() 187 except connector.DatabaseNotReadyError: 188 # (message was already logged) 189 _reactor.stop() 190 return 191 192 if hasattr(signal, "SIGHUP"): 193 def sighup(*args): 194 _reactor.callLater(0, self.reconfig)

195 signal.signal(signal.SIGHUP, sighup) 196 197 # call the parent method 198 yield defer.maybeDeferred(lambda : 199 service.MultiService.startService(self)) 200 201 # give all services a chance to load the new configuration, rather than 202 # the base configuration 203 yield self.reconfigService(self.config) 204 except: 205 f = failure.Failure() 206 log.err(f, 'while starting BuildMaster') 207 _reactor.stop() 208 209 log.msg("BuildMaster is running")

210 211

212 - def stopService(self):

213 if self.db_loop: 214 self.db_loop.stop() 215 self.db_loop = None

216 217

218 - def reconfig(self):

219 # this method wraps doConfig, ensuring it is only ever called once at 220 # a time, and alerting the user if the reconfig takes too long 221 if self.reconfig_active: 222 log.msg("reconfig already active; will reconfig again after") 223 self.reconfig_requested = True 224 return 225 226 self.reconfig_active = reactor.seconds() 227 metrics.MetricCountEvent.log("loaded_config", 1) 228 229 # notify every 10 seconds that the reconfig is still going on, although 230 # reconfigs should not take that long! 231 self.reconfig_notifier = task.LoopingCall(lambda : 232 log.msg("reconfig is ongoing for %d s" % 233 (reactor.seconds() - self.reconfig_active))) 234 self.reconfig_notifier.start(10, now=False) 235 236 timer = metrics.Timer("BuildMaster.reconfig") 237 timer.start() 238 239 d = self.doReconfig() 240 241 @d.addBoth 242 def cleanup(res): 243 timer.stop() 244 self.reconfig_notifier.stop() 245 self.reconfig_notifier = None 246 self.reconfig_active = False 247 if self.reconfig_requested: 248 self.reconfig_requested = False 249 self.reconfig() 250 return res

251 252 d.addErrback(log.err, 'while reconfiguring') 253 254 return d # for tests 255 256 257 @defer.inlineCallbacks

258 - def doReconfig(self):

259 log.msg("beginning configuration update") 260 changes_made = False 261 failed = False 262 try: 263 new_config = config.MasterConfig.loadConfig(self.basedir, 264 self.configFileName) 265 changes_made = True 266 self.config = new_config 267 yield self.reconfigService(new_config) 268 269 except config.ConfigErrors, e: 270 for msg in e.errors: 271 log.msg(msg) 272 failed = True 273 274 except: 275 log.err(failure.Failure(), 'during reconfig:') 276 failed = True 277 278 if failed: 279 if changes_made: 280 log.msg("WARNING: reconfig partially applied; master " 281 "may malfunction") 282 else: 283 log.msg("reconfig aborted without making any changes") 284 else: 285 log.msg("configuration update complete")

286 287

288 - def reconfigService(self, new_config):

289 if self.config.db['db_url'] != new_config.db['db_url']: 290 config.error( 291 "Cannot change c['db']['db_url'] after the master has started", 292 ) 293 294 # adjust the db poller 295 if (self.config.db['db_poll_interval'] 296 != new_config.db['db_poll_interval']): 297 if self.db_loop: 298 self.db_loop.stop() 299 self.db_loop = None 300 poll_interval = new_config.db['db_poll_interval'] 301 if poll_interval: 302 self.db_loop = task.LoopingCall(self.pollDatabase) 303 self.db_loop.start(poll_interval, now=False) 304 305 return config.ReconfigurableServiceMixin.reconfigService(self, 306 new_config)

307 308 309 ## informational methods 310

311 - def allSchedulers(self):

312 return list(self.scheduler_manager)

313

314 - def getStatus(self):

315 """ 316 @rtype: L{buildbot.status.builder.Status} 317 """ 318 return self.status

319

320 - def getObjectId(self):

321 """ 322 Return the obejct id for this master, for associating state with the 323 master. 324 325 @returns: ID, via Deferred 326 """ 327 # try to get the cached value 328 if self._object_id is not None: 329 return defer.succeed(self._object_id) 330 331 # failing that, get it from the DB; multiple calls to this function 332 # at the same time will not hurt 333 try: 334 hostname = os.uname()[1] # only on unix 335 except AttributeError: 336 hostname = socket.getfqdn() 337 master_name = "%s:%s" % (hostname, os.path.abspath(self.basedir)) 338 339 d = self.db.state.getObjectId(master_name, 340 "buildbot.master.BuildMaster") 341 def keep(id): 342 self._object_id = id 343 return id

344 d.addCallback(keep) 345 return d 346 347 348 ## triggering methods and subscriptions 349

350 - def addChange(self, who=None, files=None, comments=None, author=None, 351 isdir=None, is_dir=None, revision=None, when=None, 352 when_timestamp=None, branch=None, category=None, revlink='', 353 properties={}, repository='', codebase=None, project='', src=None):

354 """ 355 Add a change to the buildmaster and act on it. 356 357 This is a wrapper around L{ChangesConnectorComponent.addChange} which 358 also acts on the resulting change and returns a L{Change} instance. 359 360 Note that all parameters are keyword arguments, although C{who}, 361 C{files}, and C{comments} can be specified positionally for 362 backward-compatibility. 363 364 @param author: the author of this change 365 @type author: unicode string 366 367 @param who: deprecated name for C{author} 368 369 @param files: a list of filenames that were changed 370 @type branch: list of unicode strings 371 372 @param comments: user comments on the change 373 @type branch: unicode string 374 375 @param is_dir: deprecated 376 377 @param isdir: deprecated name for C{is_dir} 378 379 @param revision: the revision identifier for this change 380 @type revision: unicode string 381 382 @param when_timestamp: when this change occurred, or the current time 383 if None 384 @type when_timestamp: datetime instance or None 385 386 @param when: deprecated name and type for C{when_timestamp} 387 @type when: integer (UNIX epoch time) or None 388 389 @param branch: the branch on which this change took place 390 @type branch: unicode string 391 392 @param category: category for this change (arbitrary use by Buildbot 393 users) 394 @type category: unicode string 395 396 @param revlink: link to a web view of this revision 397 @type revlink: unicode string 398 399 @param properties: properties to set on this change 400 @type properties: dictionary with string keys and simple values 401 (JSON-able). Note that the property source is I{not} included 402 in this dictionary. 403 404 @param repository: the repository in which this change took place 405 @type repository: unicode string 406 407 @param project: the project this change is a part of 408 @type project: unicode string 409 410 @param src: source of the change (vcs or other) 411 @type src: string 412 413 @returns: L{Change} instance via Deferred 414 """ 415 metrics.MetricCountEvent.log("added_changes", 1) 416 417 # handle translating deprecated names into new names for db.changes 418 def handle_deprec(oldname, old, newname, new, default=None, 419 converter = lambda x:x): 420 if old is not None: 421 if new is None: 422 log.msg("WARNING: change source is using deprecated " 423 "addChange parameter '%s'" % oldname) 424 return converter(old) 425 raise TypeError("Cannot provide '%s' and '%s' to addChange" 426 % (oldname, newname)) 427 if new is None: 428 new = default 429 return new

430 431 author = handle_deprec("who", who, "author", author) 432 is_dir = handle_deprec("isdir", isdir, "is_dir", is_dir, 433 default=0) 434 when_timestamp = handle_deprec("when", when, 435 "when_timestamp", when_timestamp, 436 converter=epoch2datetime) 437 438 # add a source to each property 439 for n in properties: 440 properties[n] = (properties[n], 'Change') 441 442 if codebase is None: 443 if self.config.codebaseGenerator is not None: 444 chdict = { 445 'changeid': None, 446 'author': author, 447 'files': files, 448 'comments': comments, 449 'is_dir': is_dir, 450 'revision': revision, 451 'when_timestamp': when_timestamp, 452 'branch': branch, 453 'category': category, 454 'revlink': revlink, 455 'properties': properties, 456 'repository': repository, 457 'project': project, 458 } 459 codebase = self.config.codebaseGenerator(chdict) 460 else: 461 codebase = '' 462 463 d = defer.succeed(None) 464 if src: 465 # create user object, returning a corresponding uid 466 d.addCallback(lambda _ : users.createUserObject(self, author, src)) 467 468 # add the Change to the database 469 d.addCallback(lambda uid : 470 self.db.changes.addChange(author=author, files=files, 471 comments=comments, is_dir=is_dir, 472 revision=revision, 473 when_timestamp=when_timestamp, 474 branch=branch, category=category, 475 revlink=revlink, properties=properties, 476 repository=repository, codebase=codebase, 477 project=project, uid=uid)) 478 479 # convert the changeid to a Change instance 480 d.addCallback(lambda changeid : 481 self.db.changes.getChange(changeid)) 482 d.addCallback(lambda chdict : 483 changes.Change.fromChdict(self, chdict)) 484 485 def notify(change): 486 msg = u"added change %s to database" % change 487 log.msg(msg.encode('utf-8', 'replace')) 488 # only deliver messages immediately if we're not polling 489 if not self.config.db['db_poll_interval']: 490 self._change_subs.deliver(change) 491 return change 492 d.addCallback(notify) 493 return d 494

495 - def subscribeToChanges(self, callback):

496 """ 497 Request that C{callback} be called with each Change object added to the 498 cluster. 499 500 Note: this method will go away in 0.9.x 501 """ 502 return self._change_subs.subscribe(callback)

503

504 - def addBuildset(self, **kwargs):

505 """ 506 Add a buildset to the buildmaster and act on it. Interface is 507 identical to 508 L{buildbot.db.buildsets.BuildsetConnectorComponent.addBuildset}, 509 including returning a Deferred, but also potentially triggers the 510 resulting builds. 511 """ 512 d = self.db.buildsets.addBuildset(**kwargs) 513 def notify((bsid,brids)): 514 log.msg("added buildset %d to database" % bsid) 515 # note that buildset additions are only reported on this master 516 self._new_buildset_subs.deliver(bsid=bsid, **kwargs) 517 # only deliver messages immediately if we're not polling 518 if not self.config.db['db_poll_interval']: 519 for bn, brid in brids.iteritems(): 520 self.buildRequestAdded(bsid=bsid, brid=brid, 521 buildername=bn) 522 return (bsid,brids)

523 d.addCallback(notify) 524 return d 525

526 - def subscribeToBuildsets(self, callback):

527 """ 528 Request that C{callback(bsid=bsid, ssid=ssid, reason=reason, 529 properties=properties, builderNames=builderNames, 530 external_idstring=external_idstring)} be called whenever a buildset is 531 added. Properties is a dictionary as expected for 532 L{BuildsetsConnectorComponent.addBuildset}. 533 534 Note that this only works for buildsets added on this master. 535 536 Note: this method will go away in 0.9.x 537 """ 538 return self._new_buildset_subs.subscribe(callback)

539 540 @defer.inlineCallbacks

541 - def maybeBuildsetComplete(self, bsid):

542 """ 543 Instructs the master to check whether the buildset is complete, 544 and notify appropriately if it is. 545 546 Note that buildset completions are only reported on the master 547 on which the last build request completes. 548 """ 549 brdicts = yield self.db.buildrequests.getBuildRequests( 550 bsid=bsid, complete=False) 551 552 # if there are incomplete buildrequests, bail out 553 if brdicts: 554 return 555 556 brdicts = yield self.db.buildrequests.getBuildRequests(bsid=bsid) 557 558 # figure out the overall results of the buildset 559 cumulative_results = SUCCESS 560 for brdict in brdicts: 561 if brdict['results'] not in (SUCCESS, WARNINGS): 562 cumulative_results = FAILURE 563 564 # mark it as completed in the database 565 yield self.db.buildsets.completeBuildset(bsid, cumulative_results) 566 567 # and deliver to any listeners 568 self._buildsetComplete(bsid, cumulative_results)

569

570 - def _buildsetComplete(self, bsid, results):

571 self._complete_buildset_subs.deliver(bsid, results)

572

573 - def subscribeToBuildsetCompletions(self, callback):

574 """ 575 Request that C{callback(bsid, result)} be called whenever a 576 buildset is complete. 577 578 Note: this method will go away in 0.9.x 579 """ 580 return self._complete_buildset_subs.subscribe(callback)

581

582 - def buildRequestAdded(self, bsid, brid, buildername):

583 """ 584 Notifies the master that a build request is available to be claimed; 585 this may be a brand new build request, or a build request that was 586 previously claimed and unclaimed through a timeout or other calamity. 587 588 @param bsid: containing buildset id 589 @param brid: buildrequest ID 590 @param buildername: builder named by the build request 591 """ 592 self._new_buildrequest_subs.deliver( 593 dict(bsid=bsid, brid=brid, buildername=buildername))

594

595 - def subscribeToBuildRequests(self, callback):

596 """ 597 Request that C{callback} be invoked with a dictionary with keys C{brid} 598 (the build request id), C{bsid} (buildset id) and C{buildername} 599 whenever a new build request is added to the database. Note that, due 600 to the delayed nature of subscriptions, the build request may already 601 be claimed by the time C{callback} is invoked. 602 603 Note: this method will go away in 0.9.x 604 """ 605 return self._new_buildrequest_subs.subscribe(callback)

606 607 608 ## database polling 609

610 - def pollDatabase(self):

611 # poll each of the tables that can indicate new, actionable stuff for 612 # this buildmaster to do. This is used in a TimerService, so returning 613 # a Deferred means that we won't run two polling operations 614 # simultaneously. Each particular poll method handles errors itself, 615 # although catastrophic errors are handled here 616 d = defer.gatherResults([ 617 self.pollDatabaseChanges(), 618 self.pollDatabaseBuildRequests(), 619 # also unclaim 620 ]) 621 d.addErrback(log.err, 'while polling database') 622 return d

623 624 _last_processed_change = None 625 @defer.inlineCallbacks

626 - def pollDatabaseChanges(self):

627 # Older versions of Buildbot had each scheduler polling the database 628 # independently, and storing a "last_processed" state indicating the 629 # last change it had processed. This had the advantage of allowing 630 # schedulers to pick up changes that arrived in the database while 631 # the scheduler was not running, but was horribly inefficient. 632 633 # This version polls the database on behalf of the schedulers, using a 634 # similar state at the master level. 635 636 timer = metrics.Timer("BuildMaster.pollDatabaseChanges()") 637 timer.start() 638 639 need_setState = False 640 641 # get the last processed change id 642 if self._last_processed_change is None: 643 self._last_processed_change = \ 644 yield self._getState('last_processed_change') 645 646 # if it's still None, assume we've processed up to the latest changeid 647 if self._last_processed_change is None: 648 lpc = yield self.db.changes.getLatestChangeid() 649 # if there *are* no changes, count the last as '0' so that we don't 650 # skip the first change 651 if lpc is None: 652 lpc = 0 653 self._last_processed_change = lpc 654 655 need_setState = True 656 657 if self._last_processed_change is None: 658 timer.stop() 659 return 660 661 while True: 662 changeid = self._last_processed_change + 1 663 chdict = yield self.db.changes.getChange(changeid) 664 665 # if there's no such change, we've reached the end and can 666 # stop polling 667 if not chdict: 668 break 669 670 change = yield changes.Change.fromChdict(self, chdict) 671 672 self._change_subs.deliver(change) 673 674 self._last_processed_change = changeid 675 need_setState = True 676 677 # write back the updated state, if it's changed 678 if need_setState: 679 yield self._setState('last_processed_change', 680 self._last_processed_change) 681 timer.stop()

682 683 _last_unclaimed_brids_set = None 684 _last_claim_cleanup = 0 685 @defer.inlineCallbacks

686 - def pollDatabaseBuildRequests(self):

687 # deal with cleaning up unclaimed requests, and (if necessary) 688 # requests from a previous instance of this master 689 timer = metrics.Timer("BuildMaster.pollDatabaseBuildRequests()") 690 timer.start() 691 692 # cleanup unclaimed builds 693 since_last_cleanup = reactor.seconds() - self._last_claim_cleanup 694 if since_last_cleanup < self.RECLAIM_BUILD_INTERVAL: 695 unclaimed_age = (self.RECLAIM_BUILD_INTERVAL 696 * self.UNCLAIMED_BUILD_FACTOR) 697 yield self.db.buildrequests.unclaimExpiredRequests(unclaimed_age) 698 699 self._last_claim_cleanup = reactor.seconds() 700 701 # _last_unclaimed_brids_set tracks the state of unclaimed build 702 # requests; whenever it sees a build request which was not claimed on 703 # the last poll, it notifies the subscribers. It only tracks that 704 # state within the master instance, though; on startup, it notifies for 705 # all unclaimed requests in the database. 706 707 last_unclaimed = self._last_unclaimed_brids_set or set() 708 if len(last_unclaimed) > self.WARNING_UNCLAIMED_COUNT: 709 log.msg("WARNING: %d unclaimed buildrequests - is a scheduler " 710 "producing builds for which no builder is running?" 711 % len(last_unclaimed)) 712 713 # get the current set of unclaimed buildrequests 714 now_unclaimed_brdicts = \ 715 yield self.db.buildrequests.getBuildRequests(claimed=False) 716 now_unclaimed = set([ brd['brid'] for brd in now_unclaimed_brdicts ]) 717 718 # and store that for next time 719 self._last_unclaimed_brids_set = now_unclaimed 720 721 # see what's new, and notify if anything is 722 new_unclaimed = now_unclaimed - last_unclaimed 723 if new_unclaimed: 724 brdicts = dict((brd['brid'], brd) for brd in now_unclaimed_brdicts) 725 for brid in new_unclaimed: 726 brd = brdicts[brid] 727 self.buildRequestAdded(brd['buildsetid'], brd['brid'], 728 brd['buildername']) 729 timer.stop()

730 731 ## state maintenance (private) 732

733 - def _getState(self, name, default=None):

734 "private wrapper around C{self.db.state.getState}" 735 d = self.getObjectId() 736 def get(objectid): 737 return self.db.state.getState(objectid, name, default)

738 d.addCallback(get) 739 return d 740

741 - def _setState(self, name, value):

742 "private wrapper around C{self.db.state.setState}" 743 d = self.getObjectId() 744 def set(objectid): 745 return self.db.state.setState(objectid, name, value)

746 d.addCallback(set) 747 return d 748

749 -class Control:

750 implements(interfaces.IControl) 751

752 - def __init__(self, master):

753 self.master = master

754

755 - def addChange(self, change):

756 self.master.addChange(change)

757

758 - def addBuildset(self, **kwargs):

759 return self.master.addBuildset(**kwargs)

760

761 - def getBuilder(self, name):

762 b = self.master.botmaster.builders[name] 763 return BuilderControl(b, self)

764 765 components.registerAdapter(Control, BuildMaster, interfaces.IControl) 766

Source Code for Module buildbot.master