buildbot.schedulers.base

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 import logging 17 from zope.interface import implements 18 from twisted.python import failure, log 19 from twisted.application import service 20 from twisted.internet import defer 21 from buildbot.process.properties import Properties 22 from buildbot.util import ComparableMixin 23 from buildbot import config, interfaces 24 from buildbot.util.state import StateMixin

25 26 -class BaseScheduler(service.MultiService, ComparableMixin, StateMixin):

27 """ 28 Base class for all schedulers; this provides the equipment to manage 29 reconfigurations and to handle basic scheduler state. It also provides 30 utility methods to begin various sorts of builds. 31 32 Subclasses should add any configuration-derived attributes to 33 C{base.Scheduler.compare_attrs}. 34 """ 35 36 implements(interfaces.IScheduler) 37 38 DefaultCodebases = {'':{}} 39 40 compare_attrs = ('name', 'builderNames', 'properties', 'codebases') 41

42 - def __init__(self, name, builderNames, properties, 43 codebases = DefaultCodebases):

44 """ 45 Initialize a Scheduler. 46 47 @param name: name of this scheduler (used as a key for state) 48 @type name: unicode 49 50 @param builderNames: list of builders this scheduler may start 51 @type builderNames: list of unicode 52 53 @param properties: properties to add to builds triggered by this 54 scheduler 55 @type properties: dictionary 56 57 @param codebases: codebases that are necessary to process the changes 58 @type codebases: dict with following struct: 59 key: '<codebase>' 60 value: {'repository':'<repo>', 'branch':'<br>', 'revision:'<rev>'} 61 62 @param consumeChanges: true if this scheduler wishes to be informed 63 about the addition of new changes. Defaults to False. This should 64 be passed explicitly from subclasses to indicate their interest in 65 consuming changes. 66 @type consumeChanges: boolean 67 """ 68 service.MultiService.__init__(self) 69 self.name = name 70 "name of this scheduler; used to identify replacements on reconfig" 71 72 ok = True 73 if not isinstance(builderNames, (list, tuple)): 74 ok = False 75 else: 76 for b in builderNames: 77 if not isinstance(b, basestring): 78 ok = False 79 if not ok: 80 config.error( 81 "The builderNames argument to a scheduler must be a list " 82 "of Builder names.") 83 84 self.builderNames = builderNames 85 "list of builder names to start in each buildset" 86 87 self.properties = Properties() 88 "properties that are contributed to each buildset" 89 self.properties.update(properties, "Scheduler") 90 self.properties.setProperty("scheduler", name, "Scheduler") 91 92 self.objectid = None 93 94 self.master = None 95 96 # Set the codebases that are necessary to process the changes 97 # These codebases will always result in a sourcestamp with or without changes 98 if codebases is not None: 99 if not isinstance(codebases, dict): 100 config.error("Codebases must be a dict of dicts") 101 for codebase, codebase_attrs in codebases.iteritems(): 102 if not isinstance(codebase_attrs, dict): 103 config.error("Codebases must be a dict of dicts") 104 if (codebases != BaseScheduler.DefaultCodebases and 105 'repository' not in codebase_attrs): 106 config.error("The key 'repository' is mandatory in codebases") 107 else: 108 config.error("Codebases cannot be None") 109 110 self.codebases = codebases 111 112 # internal variables 113 self._change_subscription = None 114 self._change_consumption_lock = defer.DeferredLock()

115 116 ## service handling 117

118 - def startService(self):

119 service.MultiService.startService(self)

120

121 - def findNewSchedulerInstance(self, new_config):

122 return new_config.schedulers[self.name] # should exist!

123

124 - def stopService(self):

125 d = defer.maybeDeferred(self._stopConsumingChanges) 126 d.addCallback(lambda _ : service.MultiService.stopService(self)) 127 return d

128 129 130 ## status queries 131 132 # TODO: these aren't compatible with distributed schedulers 133

134 - def listBuilderNames(self):

135 "Returns the list of builder names" 136 return self.builderNames

137

138 - def getPendingBuildTimes(self):

139 "Returns a list of the next times that builds are scheduled, if known." 140 return []

141 142 ## change handling 143

144 - def startConsumingChanges(self, fileIsImportant=None, change_filter=None, 145 onlyImportant=False):

146 """ 147 Subclasses should call this method from startService to register to 148 receive changes. The BaseScheduler class will take care of filtering 149 the changes (using change_filter) and (if fileIsImportant is not None) 150 classifying them. See L{gotChange}. Returns a Deferred. 151 152 @param fileIsImportant: a callable provided by the user to distinguish 153 important and unimportant changes 154 @type fileIsImportant: callable 155 156 @param change_filter: a filter to determine which changes are even 157 considered by this scheduler, or C{None} to consider all changes 158 @type change_filter: L{buildbot.changes.filter.ChangeFilter} instance 159 160 @param onlyImportant: If True, only important changes, as specified by 161 fileIsImportant, will be added to the buildset. 162 @type onlyImportant: boolean 163 164 """ 165 assert fileIsImportant is None or callable(fileIsImportant) 166 167 # register for changes with master 168 assert not self._change_subscription 169 def changeCallback(change): 170 # ignore changes delivered while we're not running 171 if not self._change_subscription: 172 return 173 174 if change_filter and not change_filter.filter_change(change): 175 return 176 if change.codebase not in self.codebases: 177 log.msg('change contains codebase %s that is not processed by' 178 ' scheduler %s' % (change.codebase, self.name), 179 logLevel=logging.DEBUG) 180 return 181 if fileIsImportant: 182 try: 183 important = fileIsImportant(change) 184 if not important and onlyImportant: 185 return 186 except: 187 log.err(failure.Failure(), 188 'in fileIsImportant check for %s' % change) 189 return 190 else: 191 important = True 192 193 # use change_consumption_lock to ensure the service does not stop 194 # while this change is being processed 195 d = self._change_consumption_lock.acquire() 196 d.addCallback(lambda _ : self.gotChange(change, important)) 197 def release(x): 198 self._change_consumption_lock.release()

199 d.addBoth(release) 200 d.addErrback(log.err, 'while processing change')

201 self._change_subscription = self.master.subscribeToChanges(changeCallback) 202 203 return defer.succeed(None) 204

205 - def _stopConsumingChanges(self):

206 # (note: called automatically in stopService) 207 208 # acquire the lock change consumption lock to ensure that any change 209 # consumption is complete before we are done stopping consumption 210 d = self._change_consumption_lock.acquire() 211 def stop(x): 212 if self._change_subscription: 213 self._change_subscription.unsubscribe() 214 self._change_subscription = None 215 self._change_consumption_lock.release()

216 d.addBoth(stop) 217 return d 218

219 - def gotChange(self, change, important):

220 """ 221 Called when a change is received; returns a Deferred. If the 222 C{fileIsImportant} parameter to C{startConsumingChanges} was C{None}, 223 then all changes are considered important. 224 The C{codebase} of the change has always an entry in the C{codebases} 225 dictionary of the scheduler. 226 227 @param change: the new change object 228 @type change: L{buildbot.changes.changes.Change} instance 229 @param important: true if this is an important change, according to 230 C{fileIsImportant}. 231 @type important: boolean 232 @returns: Deferred 233 """ 234 raise NotImplementedError

235 236 ## starting bulids 237 238 @defer.inlineCallbacks

239 - def addBuildsetForLatest(self, reason='', external_idstring=None, 240 branch=None, repository='', project='', 241 builderNames=None, properties=None):

242 """ 243 Add a buildset for the 'latest' source in the given branch, 244 repository, and project. This will create a relative sourcestamp for 245 the buildset. 246 247 This method will add any properties provided to the scheduler 248 constructor to the buildset, and will call the master's addBuildset 249 method with the appropriate parameters. 250 251 @param reason: reason for this buildset 252 @type reason: unicode string 253 @param external_idstring: external identifier for this buildset, or None 254 @param branch: branch to build (note that None often has a special meaning) 255 @param repository: repository name for sourcestamp 256 @param project: project name for sourcestamp 257 @param builderNames: builders to name in the buildset (defaults to 258 C{self.builderNames}) 259 @param properties: a properties object containing initial properties for 260 the buildset 261 @type properties: L{buildbot.process.properties.Properties} 262 @returns: (buildset ID, buildrequest IDs) via Deferred 263 """ 264 # Define setid for this set of changed repositories 265 setid = yield self.master.db.sourcestampsets.addSourceStampSet() 266 267 # add a sourcestamp for each codebase 268 for codebase, cb_info in self.codebases.iteritems(): 269 ss_repository = cb_info.get('repository', repository) 270 ss_branch = cb_info.get('branch', branch) 271 ss_revision = cb_info.get('revision', None) 272 yield self.master.db.sourcestamps.addSourceStamp( 273 codebase=codebase, 274 repository=ss_repository, 275 branch=ss_branch, 276 revision=ss_revision, 277 project=project, 278 changeids=set(), 279 sourcestampsetid=setid) 280 281 bsid,brids = yield self.addBuildsetForSourceStamp( 282 setid=setid, reason=reason, 283 external_idstring=external_idstring, 284 builderNames=builderNames, 285 properties=properties) 286 287 defer.returnValue((bsid,brids))

288 289 290 @defer.inlineCallbacks

291 - def addBuildsetForSourceStampDetails(self, reason='', external_idstring=None, 292 branch=None, repository='', project='', revision=None, 293 builderNames=None, properties=None):

294 """ 295 Given details about the source code to build, create a source stamp and 296 then add a buildset for it. 297 298 @param reason: reason for this buildset 299 @type reason: unicode string 300 @param external_idstring: external identifier for this buildset, or None 301 @param branch: branch to build (note that None often has a special meaning) 302 @param repository: repository name for sourcestamp 303 @param project: project name for sourcestamp 304 @param revision: revision to build - default is latest 305 @param builderNames: builders to name in the buildset (defaults to 306 C{self.builderNames}) 307 @param properties: a properties object containing initial properties for 308 the buildset 309 @type properties: L{buildbot.process.properties.Properties} 310 @returns: (buildset ID, buildrequest IDs) via Deferred 311 """ 312 # Define setid for this set of changed repositories 313 setid = yield self.master.db.sourcestampsets.addSourceStampSet() 314 315 yield self.master.db.sourcestamps.addSourceStamp( 316 branch=branch, revision=revision, repository=repository, 317 project=project, sourcestampsetid=setid) 318 319 rv = yield self.addBuildsetForSourceStamp( 320 setid=setid, reason=reason, 321 external_idstring=external_idstring, 322 builderNames=builderNames, 323 properties=properties) 324 defer.returnValue(rv)

325 326 327 @defer.inlineCallbacks

328 - def addBuildsetForSourceStampSetDetails(self, reason, sourcestamps, 329 properties, builderNames=None):

330 if sourcestamps is None: 331 sourcestamps = {} 332 333 # Define new setid for this set of sourcestamps 334 new_setid = yield self.master.db.sourcestampsets.addSourceStampSet() 335 336 # Merge codebases with the passed list of sourcestamps 337 # This results in a new sourcestamp for each codebase 338 for codebase in self.codebases: 339 ss = self.codebases[codebase].copy() 340 # apply info from passed sourcestamps onto the configured default 341 # sourcestamp attributes for this codebase. 342 ss.update(sourcestamps.get(codebase,{})) 343 344 # add sourcestamp to the new setid 345 yield self.master.db.sourcestamps.addSourceStamp( 346 codebase=codebase, 347 repository=ss.get('repository', None), 348 branch=ss.get('branch', None), 349 revision=ss.get('revision', None), 350 project=ss.get('project', ''), 351 changeids=[c['number'] for c in ss.get('changes', [])], 352 patch_body=ss.get('patch_body', None), 353 patch_level=ss.get('patch_level', None), 354 patch_author=ss.get('patch_author', None), 355 patch_comment=ss.get('patch_comment', None), 356 sourcestampsetid=new_setid) 357 358 rv = yield self.addBuildsetForSourceStamp( 359 setid=new_setid, reason=reason, 360 properties=properties, 361 builderNames=builderNames) 362 363 defer.returnValue(rv)

364 365 366 @defer.inlineCallbacks

367 - def addBuildsetForChanges(self, reason='', external_idstring=None, 368 changeids=[], builderNames=None, properties=None):

369 changesByCodebase = {} 370 371 def get_last_change_for_codebase(codebase): 372 return max(changesByCodebase[codebase],key = lambda change: change["changeid"])

373 374 # Define setid for this set of changed repositories 375 setid = yield self.master.db.sourcestampsets.addSourceStampSet() 376 377 # Changes are retrieved from database and grouped by their codebase 378 for changeid in changeids: 379 chdict = yield self.master.db.changes.getChange(changeid) 380 # group change by codebase 381 changesByCodebase.setdefault(chdict["codebase"], []).append(chdict) 382 383 for codebase in self.codebases: 384 args = {'codebase': codebase, 'sourcestampsetid': setid } 385 if codebase not in changesByCodebase: 386 # codebase has no changes 387 # create a sourcestamp that has no changes 388 args['repository'] = self.codebases[codebase]['repository'] 389 args['branch'] = self.codebases[codebase].get('branch', None) 390 args['revision'] = self.codebases[codebase].get('revision', None) 391 args['changeids'] = set() 392 args['project'] = '' 393 else: 394 #codebase has changes 395 args['changeids'] = [c["changeid"] for c in changesByCodebase[codebase]] 396 lastChange = get_last_change_for_codebase(codebase) 397 for key in ['repository', 'branch', 'revision', 'project']: 398 args[key] = lastChange[key] 399 400 yield self.master.db.sourcestamps.addSourceStamp(**args) 401 402 # add one buildset, this buildset is connected to the sourcestamps by the setid 403 bsid,brids = yield self.addBuildsetForSourceStamp( setid=setid, 404 reason=reason, external_idstring=external_idstring, 405 builderNames=builderNames, properties=properties) 406 407 defer.returnValue((bsid,brids)) 408 409 @defer.inlineCallbacks

410 - def addBuildsetForSourceStamp(self, ssid=None, setid=None, reason='', external_idstring=None, 411 properties=None, builderNames=None):

412 """ 413 Add a buildset for the given, already-existing sourcestamp. 414 415 This method will add any properties provided to the scheduler 416 constructor to the buildset, and will call the master's 417 L{BuildMaster.addBuildset} method with the appropriate parameters, and 418 return the same result. 419 420 @param reason: reason for this buildset 421 @type reason: unicode string 422 @param external_idstring: external identifier for this buildset, or None 423 @param properties: a properties object containing initial properties for 424 the buildset 425 @type properties: L{buildbot.process.properties.Properties} 426 @param builderNames: builders to name in the buildset (defaults to 427 C{self.builderNames}) 428 @param setid: idenitification of a set of sourcestamps 429 @returns: (buildset ID, buildrequest IDs) via Deferred 430 """ 431 assert (ssid is None and setid is not None) \ 432 or (ssid is not None and setid is None), "pass a single sourcestamp OR set not both" 433 434 # combine properties 435 if properties: 436 properties.updateFromProperties(self.properties) 437 else: 438 properties = self.properties 439 440 # apply the default builderNames 441 if not builderNames: 442 builderNames = self.builderNames 443 444 # translate properties object into a dict as required by the 445 # addBuildset method 446 properties_dict = properties.asDict() 447 448 if setid == None: 449 if ssid is not None: 450 ssdict = yield self.master.db.sourcestamps.getSourceStamp(ssid) 451 setid = ssdict['sourcestampsetid'] 452 else: 453 # no sourcestamp and no sets 454 yield None 455 456 rv = yield self.master.addBuildset(sourcestampsetid=setid, 457 reason=reason, properties=properties_dict, 458 builderNames=builderNames, 459 external_idstring=external_idstring) 460 defer.returnValue(rv)

461

Source Code for Module buildbot.schedulers.base