buildbot.schedulers.base

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 from zope.interface import implements 17 from twisted.python import failure, log 18 from twisted.application import service 19 from twisted.internet import defer 20 from buildbot.process.properties import Properties 21 from buildbot.util import ComparableMixin 22 from buildbot import config, interfaces 23 from buildbot.util.state import StateMixin

24 25 -class BaseScheduler(service.MultiService, ComparableMixin, StateMixin):

26 """ 27 Base class for all schedulers; this provides the equipment to manage 28 reconfigurations and to handle basic scheduler state. It also provides 29 utility methods to begin various sorts of builds. 30 31 Subclasses should add any configuration-derived attributes to 32 C{base.Scheduler.compare_attrs}. 33 """ 34 35 implements(interfaces.IScheduler) 36 37 DefaultCodebases = {'':{}} 38 39 compare_attrs = ('name', 'builderNames', 'properties', 'codebases') 40

41 - def __init__(self, name, builderNames, properties, 42 codebases = DefaultCodebases):

43 """ 44 Initialize a Scheduler. 45 46 @param name: name of this scheduler (used as a key for state) 47 @type name: unicode 48 49 @param builderNames: list of builders this scheduler may start 50 @type builderNames: list of unicode 51 52 @param properties: properties to add to builds triggered by this 53 scheduler 54 @type properties: dictionary 55 56 @param codebases: codebases that are necessary to process the changes 57 @type codebases: dict with following struct: 58 key: '<codebase>' 59 value: {'repository':'<repo>', 'branch':'<br>', 'revision:'<rev>'} 60 61 @param consumeChanges: true if this scheduler wishes to be informed 62 about the addition of new changes. Defaults to False. This should 63 be passed explicitly from subclasses to indicate their interest in 64 consuming changes. 65 @type consumeChanges: boolean 66 """ 67 service.MultiService.__init__(self) 68 self.name = name 69 "name of this scheduler; used to identify replacements on reconfig" 70 71 ok = True 72 if not isinstance(builderNames, (list, tuple)): 73 ok = False 74 else: 75 for b in builderNames: 76 if not isinstance(b, basestring): 77 ok = False 78 if not ok: 79 config.error( 80 "The builderNames argument to a scheduler must be a list " 81 "of Builder names.") 82 83 self.builderNames = builderNames 84 "list of builder names to start in each buildset" 85 86 self.properties = Properties() 87 "properties that are contributed to each buildset" 88 self.properties.update(properties, "Scheduler") 89 self.properties.setProperty("scheduler", name, "Scheduler") 90 91 self.objectid = None 92 93 self.master = None 94 95 # Set the codebases that are necessary to process the changes 96 # These codebases will always result in a sourcestamp with or without changes 97 if codebases is not None: 98 if not isinstance(codebases, dict): 99 config.error("Codebases must be a dict of dicts") 100 for codebase, codebase_attrs in codebases.iteritems(): 101 if not isinstance(codebase_attrs, dict): 102 config.error("Codebases must be a dict of dicts") 103 if (codebases != BaseScheduler.DefaultCodebases and 104 'repository' not in codebase_attrs): 105 config.error("The key 'repository' is mandatory in codebases") 106 else: 107 config.error("Codebases cannot be None") 108 109 self.codebases = codebases 110 111 # internal variables 112 self._change_subscription = None 113 self._change_consumption_lock = defer.DeferredLock()

114 115 ## service handling 116

117 - def startService(self):

118 service.MultiService.startService(self)

119

120 - def findNewSchedulerInstance(self, new_config):

121 return new_config.schedulers[self.name] # should exist!

122

123 - def stopService(self):

124 d = defer.maybeDeferred(self._stopConsumingChanges) 125 d.addCallback(lambda _ : service.MultiService.stopService(self)) 126 return d

127 128 129 ## status queries 130 131 # TODO: these aren't compatible with distributed schedulers 132

133 - def listBuilderNames(self):

134 "Returns the list of builder names" 135 return self.builderNames

136

137 - def getPendingBuildTimes(self):

138 "Returns a list of the next times that builds are scheduled, if known." 139 return []

140 141 ## change handling 142

143 - def startConsumingChanges(self, fileIsImportant=None, change_filter=None, 144 onlyImportant=False):

145 """ 146 Subclasses should call this method from startService to register to 147 receive changes. The BaseScheduler class will take care of filtering 148 the changes (using change_filter) and (if fileIsImportant is not None) 149 classifying them. See L{gotChange}. Returns a Deferred. 150 151 @param fileIsImportant: a callable provided by the user to distinguish 152 important and unimportant changes 153 @type fileIsImportant: callable 154 155 @param change_filter: a filter to determine which changes are even 156 considered by this scheduler, or C{None} to consider all changes 157 @type change_filter: L{buildbot.changes.filter.ChangeFilter} instance 158 159 @param onlyImportant: If True, only important changes, as specified by 160 fileIsImportant, will be added to the buildset. 161 @type onlyImportant: boolean 162 163 """ 164 assert fileIsImportant is None or callable(fileIsImportant) 165 166 # register for changes with master 167 assert not self._change_subscription 168 def changeCallback(change): 169 # ignore changes delivered while we're not running 170 if not self._change_subscription: 171 return 172 173 if change_filter and not change_filter.filter_change(change): 174 return 175 if change.codebase not in self.codebases: 176 log.msg(format='change contains codebase %(codebase)s that is' 177 'not processed by scheduler %(scheduler)s', 178 codebase=change.codebase, name=self.name) 179 return 180 if fileIsImportant: 181 try: 182 important = fileIsImportant(change) 183 if not important and onlyImportant: 184 return 185 except: 186 log.err(failure.Failure(), 187 'in fileIsImportant check for %s' % change) 188 return 189 else: 190 important = True 191 192 # use change_consumption_lock to ensure the service does not stop 193 # while this change is being processed 194 d = self._change_consumption_lock.run(self.gotChange, change, important) 195 d.addErrback(log.err, 'while processing change')

196 self._change_subscription = self.master.subscribeToChanges(changeCallback) 197 198 return defer.succeed(None)

199

200 - def _stopConsumingChanges(self):

201 # (note: called automatically in stopService) 202 203 # acquire the lock change consumption lock to ensure that any change 204 # consumption is complete before we are done stopping consumption 205 def stop(): 206 if self._change_subscription: 207 self._change_subscription.unsubscribe() 208 self._change_subscription = None

209 return self._change_consumption_lock.run(stop) 210

211 - def gotChange(self, change, important):

212 """ 213 Called when a change is received; returns a Deferred. If the 214 C{fileIsImportant} parameter to C{startConsumingChanges} was C{None}, 215 then all changes are considered important. 216 The C{codebase} of the change has always an entry in the C{codebases} 217 dictionary of the scheduler. 218 219 @param change: the new change object 220 @type change: L{buildbot.changes.changes.Change} instance 221 @param important: true if this is an important change, according to 222 C{fileIsImportant}. 223 @type important: boolean 224 @returns: Deferred 225 """ 226 raise NotImplementedError

227 228 ## starting bulids 229 230 @defer.inlineCallbacks

231 - def addBuildsetForLatest(self, reason='', external_idstring=None, 232 branch=None, repository='', project='', 233 builderNames=None, properties=None):

234 """ 235 Add a buildset for the 'latest' source in the given branch, 236 repository, and project. This will create a relative sourcestamp for 237 the buildset. 238 239 This method will add any properties provided to the scheduler 240 constructor to the buildset, and will call the master's addBuildset 241 method with the appropriate parameters. 242 243 @param reason: reason for this buildset 244 @type reason: unicode string 245 @param external_idstring: external identifier for this buildset, or None 246 @param branch: branch to build (note that None often has a special meaning) 247 @param repository: repository name for sourcestamp 248 @param project: project name for sourcestamp 249 @param builderNames: builders to name in the buildset (defaults to 250 C{self.builderNames}) 251 @param properties: a properties object containing initial properties for 252 the buildset 253 @type properties: L{buildbot.process.properties.Properties} 254 @returns: (buildset ID, buildrequest IDs) via Deferred 255 """ 256 # Define setid for this set of changed repositories 257 setid = yield self.master.db.sourcestampsets.addSourceStampSet() 258 259 # add a sourcestamp for each codebase 260 for codebase, cb_info in self.codebases.iteritems(): 261 ss_repository = cb_info.get('repository', repository) 262 ss_branch = cb_info.get('branch', branch) 263 ss_revision = cb_info.get('revision', None) 264 yield self.master.db.sourcestamps.addSourceStamp( 265 codebase=codebase, 266 repository=ss_repository, 267 branch=ss_branch, 268 revision=ss_revision, 269 project=project, 270 changeids=set(), 271 sourcestampsetid=setid) 272 273 bsid,brids = yield self.addBuildsetForSourceStamp( 274 setid=setid, reason=reason, 275 external_idstring=external_idstring, 276 builderNames=builderNames, 277 properties=properties) 278 279 defer.returnValue((bsid,brids))

280 281 282 @defer.inlineCallbacks

283 - def addBuildsetForSourceStampDetails(self, reason='', external_idstring=None, 284 branch=None, repository='', project='', revision=None, 285 builderNames=None, properties=None):

286 """ 287 Given details about the source code to build, create a source stamp and 288 then add a buildset for it. 289 290 @param reason: reason for this buildset 291 @type reason: unicode string 292 @param external_idstring: external identifier for this buildset, or None 293 @param branch: branch to build (note that None often has a special meaning) 294 @param repository: repository name for sourcestamp 295 @param project: project name for sourcestamp 296 @param revision: revision to build - default is latest 297 @param builderNames: builders to name in the buildset (defaults to 298 C{self.builderNames}) 299 @param properties: a properties object containing initial properties for 300 the buildset 301 @type properties: L{buildbot.process.properties.Properties} 302 @returns: (buildset ID, buildrequest IDs) via Deferred 303 """ 304 # Define setid for this set of changed repositories 305 setid = yield self.master.db.sourcestampsets.addSourceStampSet() 306 307 yield self.master.db.sourcestamps.addSourceStamp( 308 branch=branch, revision=revision, repository=repository, 309 project=project, sourcestampsetid=setid) 310 311 rv = yield self.addBuildsetForSourceStamp( 312 setid=setid, reason=reason, 313 external_idstring=external_idstring, 314 builderNames=builderNames, 315 properties=properties) 316 defer.returnValue(rv)

317 318 319 @defer.inlineCallbacks

320 - def addBuildsetForSourceStampSetDetails(self, reason, sourcestamps, 321 properties, builderNames=None):

322 if sourcestamps is None: 323 sourcestamps = {} 324 325 # Define new setid for this set of sourcestamps 326 new_setid = yield self.master.db.sourcestampsets.addSourceStampSet() 327 328 # Merge codebases with the passed list of sourcestamps 329 # This results in a new sourcestamp for each codebase 330 for codebase in self.codebases: 331 ss = self.codebases[codebase].copy() 332 # apply info from passed sourcestamps onto the configured default 333 # sourcestamp attributes for this codebase. 334 ss.update(sourcestamps.get(codebase,{})) 335 336 # add sourcestamp to the new setid 337 yield self.master.db.sourcestamps.addSourceStamp( 338 codebase=codebase, 339 repository=ss.get('repository', ''), 340 branch=ss.get('branch', None), 341 revision=ss.get('revision', None), 342 project=ss.get('project', ''), 343 changeids=[c['number'] for c in ss.get('changes', [])], 344 patch_body=ss.get('patch_body', None), 345 patch_level=ss.get('patch_level', None), 346 patch_author=ss.get('patch_author', None), 347 patch_comment=ss.get('patch_comment', None), 348 sourcestampsetid=new_setid) 349 350 rv = yield self.addBuildsetForSourceStamp( 351 setid=new_setid, reason=reason, 352 properties=properties, 353 builderNames=builderNames) 354 355 defer.returnValue(rv)

356 357 358 @defer.inlineCallbacks

359 - def addBuildsetForChanges(self, reason='', external_idstring=None, 360 changeids=[], builderNames=None, properties=None):

361 changesByCodebase = {} 362 363 def get_last_change_for_codebase(codebase): 364 return max(changesByCodebase[codebase],key = lambda change: change["changeid"])

365 366 # Define setid for this set of changed repositories 367 setid = yield self.master.db.sourcestampsets.addSourceStampSet() 368 369 # Changes are retrieved from database and grouped by their codebase 370 for changeid in changeids: 371 chdict = yield self.master.db.changes.getChange(changeid) 372 # group change by codebase 373 changesByCodebase.setdefault(chdict["codebase"], []).append(chdict) 374 375 for codebase in self.codebases: 376 args = {'codebase': codebase, 'sourcestampsetid': setid } 377 if codebase not in changesByCodebase: 378 # codebase has no changes 379 # create a sourcestamp that has no changes 380 args['repository'] = self.codebases[codebase]['repository'] 381 args['branch'] = self.codebases[codebase].get('branch', None) 382 args['revision'] = self.codebases[codebase].get('revision', None) 383 args['changeids'] = set() 384 args['project'] = '' 385 else: 386 #codebase has changes 387 args['changeids'] = [c["changeid"] for c in changesByCodebase[codebase]] 388 lastChange = get_last_change_for_codebase(codebase) 389 for key in ['repository', 'branch', 'revision', 'project']: 390 args[key] = lastChange[key] 391 392 yield self.master.db.sourcestamps.addSourceStamp(**args) 393 394 # add one buildset, this buildset is connected to the sourcestamps by the setid 395 bsid,brids = yield self.addBuildsetForSourceStamp( setid=setid, 396 reason=reason, external_idstring=external_idstring, 397 builderNames=builderNames, properties=properties) 398 399 defer.returnValue((bsid,brids)) 400 401 @defer.inlineCallbacks

402 - def addBuildsetForSourceStamp(self, ssid=None, setid=None, reason='', external_idstring=None, 403 properties=None, builderNames=None):

404 """ 405 Add a buildset for the given, already-existing sourcestamp. 406 407 This method will add any properties provided to the scheduler 408 constructor to the buildset, and will call the master's 409 L{BuildMaster.addBuildset} method with the appropriate parameters, and 410 return the same result. 411 412 @param reason: reason for this buildset 413 @type reason: unicode string 414 @param external_idstring: external identifier for this buildset, or None 415 @param properties: a properties object containing initial properties for 416 the buildset 417 @type properties: L{buildbot.process.properties.Properties} 418 @param builderNames: builders to name in the buildset (defaults to 419 C{self.builderNames}) 420 @param setid: idenitification of a set of sourcestamps 421 @returns: (buildset ID, buildrequest IDs) via Deferred 422 """ 423 assert (ssid is None and setid is not None) \ 424 or (ssid is not None and setid is None), "pass a single sourcestamp OR set not both" 425 426 # combine properties 427 if properties: 428 properties.updateFromProperties(self.properties) 429 else: 430 properties = self.properties 431 432 # apply the default builderNames 433 if not builderNames: 434 builderNames = self.builderNames 435 436 # translate properties object into a dict as required by the 437 # addBuildset method 438 properties_dict = properties.asDict() 439 440 if setid == None: 441 if ssid is not None: 442 ssdict = yield self.master.db.sourcestamps.getSourceStamp(ssid) 443 setid = ssdict['sourcestampsetid'] 444 else: 445 # no sourcestamp and no sets 446 yield None 447 448 rv = yield self.master.addBuildset(sourcestampsetid=setid, 449 reason=reason, properties=properties_dict, 450 builderNames=builderNames, 451 external_idstring=external_idstring) 452 defer.returnValue(rv)

453

Source Code for Module buildbot.schedulers.base