Package buildbot :: Package schedulers :: Module base
[frames] | no frames]

Source Code for Module buildbot.schedulers.base

  1  # This file is part of Buildbot.  Buildbot is free software: you can 
  2  # redistribute it and/or modify it under the terms of the GNU General Public 
  3  # License as published by the Free Software Foundation, version 2. 
  4  # 
  5  # This program is distributed in the hope that it will be useful, but WITHOUT 
  6  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
  7  # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more 
  8  # details. 
  9  # 
 10  # You should have received a copy of the GNU General Public License along with 
 11  # this program; if not, write to the Free Software Foundation, Inc., 51 
 12  # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 
 13  # 
 14  # Copyright Buildbot Team Members 
 15   
 16  from zope.interface import implements 
 17  from twisted.python import failure, log 
 18  from twisted.application import service 
 19  from twisted.internet import defer 
 20  from buildbot.process.properties import Properties 
 21  from buildbot.util import ComparableMixin 
 22  from buildbot.changes import changes 
 23  from buildbot import config, interfaces 
24 25 -class BaseScheduler(service.MultiService, ComparableMixin):
26 """ 27 Base class for all schedulers; this provides the equipment to manage 28 reconfigurations and to handle basic scheduler state. It also provides 29 utility methods to begin various sorts of builds. 30 31 Subclasses should add any configuration-derived attributes to 32 C{base.Scheduler.compare_attrs}. 33 """ 34 35 implements(interfaces.IScheduler) 36 37 compare_attrs = ('name', 'builderNames', 'properties') 38
39 - def __init__(self, name, builderNames, properties):
40 """ 41 Initialize a Scheduler. 42 43 @param name: name of this scheduler (used as a key for state) 44 @type name: unicode 45 46 @param builderNames: list of builders this scheduler may start 47 @type builderNames: list of unicode 48 49 @param properties: properties to add to builds triggered by this 50 scheduler 51 @type properties: dictionary 52 53 @param consumeChanges: true if this scheduler wishes to be informed 54 about the addition of new changes. Defaults to False. This should 55 be passed explicitly from subclasses to indicate their interest in 56 consuming changes. 57 @type consumeChanges: boolean 58 """ 59 service.MultiService.__init__(self) 60 self.name = name 61 "name of this scheduler; used to identify replacements on reconfig" 62 63 ok = True 64 if not isinstance(builderNames, (list, tuple)): 65 ok = False 66 else: 67 for b in builderNames: 68 if not isinstance(b, basestring): 69 ok = False 70 if not ok: 71 config.error( 72 "The builderNames argument to a scheduler must be a list " 73 "of Builder names.") 74 75 self.builderNames = builderNames 76 "list of builder names to start in each buildset" 77 78 self.properties = Properties() 79 "properties that are contributed to each buildset" 80 self.properties.update(properties, "Scheduler") 81 self.properties.setProperty("scheduler", name, "Scheduler") 82 83 self.objectid = None 84 85 self.master = None 86 87 # internal variables 88 self._change_subscription = None 89 self._change_consumption_lock = defer.DeferredLock() 90 self._objectid = None
91 92 ## service handling 93
94 - def startService(self):
95 service.MultiService.startService(self)
96
97 - def findNewSchedulerInstance(self, new_config):
98 return new_config.schedulers[self.name] # should exist!
99
100 - def stopService(self):
101 d = defer.maybeDeferred(self._stopConsumingChanges) 102 d.addCallback(lambda _ : service.MultiService.stopService(self)) 103 return d
104 105 ## state management 106 107 @defer.deferredGenerator
108 - def getState(self, *args, **kwargs):
109 """ 110 For use by subclasses; get a named state value from the scheduler's 111 state, defaulting to DEFAULT. 112 113 @param name: name of the value to retrieve 114 @param default: (optional) value to return if C{name} is not present 115 @returns: state value via a Deferred 116 @raises KeyError: if C{name} is not present and no default is given 117 @raises TypeError: if JSON parsing fails 118 """ 119 # get the objectid, if not known 120 if self._objectid is None: 121 wfd = defer.waitForDeferred( 122 self.master.db.state.getObjectId(self.name, 123 self.__class__.__name__)) 124 yield wfd 125 self._objectid = wfd.getResult() 126 127 wfd = defer.waitForDeferred( 128 self.master.db.state.getState(self._objectid, *args, **kwargs)) 129 yield wfd 130 yield wfd.getResult()
131 132 @defer.deferredGenerator
133 - def setState(self, key, value):
134 """ 135 For use by subclasses; set a named state value in the scheduler's 136 persistent state. Note that value must be json-able. 137 138 @param name: the name of the value to change 139 @param value: the value to set - must be a JSONable object 140 @param returns: Deferred 141 @raises TypeError: if JSONification fails 142 """ 143 # get the objectid, if not known 144 if self._objectid is None: 145 wfd = defer.waitForDeferred( 146 self.master.db.state.getObjectId(self.name, 147 self.__class__.__name__)) 148 yield wfd 149 self._objectid = wfd.getResult() 150 151 wfd = defer.waitForDeferred( 152 self.master.db.state.setState(self._objectid, key, value)) 153 yield wfd 154 wfd.getResult()
155 156 ## status queries 157 158 # TODO: these aren't compatible with distributed schedulers 159
160 - def listBuilderNames(self):
161 "Returns the list of builder names" 162 return self.builderNames
163
164 - def getPendingBuildTimes(self):
165 "Returns a list of the next times that builds are scheduled, if known." 166 return []
167 168 ## change handling 169
170 - def startConsumingChanges(self, fileIsImportant=None, change_filter=None, 171 onlyImportant=False):
172 """ 173 Subclasses should call this method from startService to register to 174 receive changes. The BaseScheduler class will take care of filtering 175 the changes (using change_filter) and (if fileIsImportant is not None) 176 classifying them. See L{gotChange}. Returns a Deferred. 177 178 @param fileIsImportant: a callable provided by the user to distinguish 179 important and unimportant changes 180 @type fileIsImportant: callable 181 182 @param change_filter: a filter to determine which changes are even 183 considered by this scheduler, or C{None} to consider all changes 184 @type change_filter: L{buildbot.changes.filter.ChangeFilter} instance 185 186 @param onlyImportant: If True, only important changes, as specified by 187 fileIsImportant, will be added to the buildset. 188 @type onlyImportant: boolean 189 190 """ 191 assert fileIsImportant is None or callable(fileIsImportant) 192 193 # register for changes with master 194 assert not self._change_subscription 195 def changeCallback(change): 196 # ignore changes delivered while we're not running 197 if not self._change_subscription: 198 return 199 200 if change_filter and not change_filter.filter_change(change): 201 return 202 if fileIsImportant: 203 try: 204 important = fileIsImportant(change) 205 if not important and onlyImportant: 206 return 207 except: 208 log.err(failure.Failure(), 209 'in fileIsImportant check for %s' % change) 210 return 211 else: 212 important = True 213 214 # use change_consumption_lock to ensure the service does not stop 215 # while this change is being processed 216 d = self._change_consumption_lock.acquire() 217 d.addCallback(lambda _ : self.gotChange(change, important)) 218 def release(x): 219 self._change_consumption_lock.release()
220 d.addBoth(release) 221 d.addErrback(log.err, 'while processing change')
222 self._change_subscription = self.master.subscribeToChanges(changeCallback) 223 224 return defer.succeed(None) 225
226 - def _stopConsumingChanges(self):
227 # (note: called automatically in stopService) 228 229 # acquire the lock change consumption lock to ensure that any change 230 # consumption is complete before we are done stopping consumption 231 d = self._change_consumption_lock.acquire() 232 def stop(x): 233 if self._change_subscription: 234 self._change_subscription.unsubscribe() 235 self._change_subscription = None 236 self._change_consumption_lock.release()
237 d.addBoth(stop) 238 return d 239
240 - def gotChange(self, change, important):
241 """ 242 Called when a change is received; returns a Deferred. If the 243 C{fileIsImportant} parameter to C{startConsumingChanges} was C{None}, 244 then all changes are considered important. 245 246 @param change: the new change object 247 @type change: L{buildbot.changes.changes.Change} instance 248 @param important: true if this is an important change, according to 249 C{fileIsImportant}. 250 @type important: boolean 251 @returns: Deferred 252 """ 253 raise NotImplementedError
254 255 ## starting bulids 256 257 @defer.deferredGenerator
258 - def addBuildsetForLatest(self, reason='', external_idstring=None, 259 branch=None, repository='', project='', 260 builderNames=None, properties=None):
261 """ 262 Add a buildset for the 'latest' source in the given branch, 263 repository, and project. This will create a relative sourcestamp for 264 the buildset. 265 266 This method will add any properties provided to the scheduler 267 constructor to the buildset, and will call the master's addBuildset 268 method with the appropriate parameters. 269 270 @param reason: reason for this buildset 271 @type reason: unicode string 272 @param external_idstring: external identifier for this buildset, or None 273 @param branch: branch to build (note that None often has a special meaning) 274 @param repository: repository name for sourcestamp 275 @param project: project name for sourcestamp 276 @param builderNames: builders to name in the buildset (defaults to 277 C{self.builderNames}) 278 @param properties: a properties object containing initial properties for 279 the buildset 280 @type properties: L{buildbot.process.properties.Properties} 281 @returns: (buildset ID, buildrequest IDs) via Deferred 282 """ 283 # Define setid for this set of changed repositories 284 wfd = defer.waitForDeferred(self.master.db.sourcestampsets.addSourceStampSet()) 285 yield wfd 286 setid = wfd.getResult() 287 288 wfd = defer.waitForDeferred(self.master.db.sourcestamps.addSourceStamp( 289 branch=branch, revision=None, repository=repository, 290 project=project, sourcestampsetid=setid)) 291 yield wfd 292 wfd.getResult() 293 294 wfd = defer.waitForDeferred(self.addBuildsetForSourceStamp( 295 setid=setid, reason=reason, 296 external_idstring=external_idstring, 297 builderNames=builderNames, 298 properties=properties)) 299 yield wfd 300 yield wfd.getResult()
301 302 @defer.deferredGenerator
303 - def addBuildsetForChanges(self, reason='', external_idstring=None, 304 changeids=[], builderNames=None, properties=None):
305 """ 306 Add a buildset for the combination of the given changesets, creating 307 a sourcestamp based on those changes. The sourcestamp for the buildset 308 will reference all of the indicated changes. 309 310 This method will add any properties provided to the scheduler 311 constructor to the buildset, and will call the master's addBuildset 312 method with the appropriate parameters. 313 314 @param reason: reason for this buildset 315 @type reason: unicode string 316 @param external_idstring: external identifier for this buildset, or None 317 @param changeids: nonempty list of changes to include in this buildset 318 @param builderNames: builders to name in the buildset (defaults to 319 C{self.builderNames}) 320 @param properties: a properties object containing initial properties for 321 the buildset 322 @type properties: L{buildbot.process.properties.Properties} 323 @returns: (buildset ID, buildrequest IDs) via Deferred 324 """ 325 assert changeids is not [] 326 327 # attributes for this sourcestamp will be based on the most recent 328 # change, so fetch the change with the highest id 329 wfd = defer.waitForDeferred(self.master.db.changes.getChange(max(changeids))) 330 yield wfd 331 chdict = wfd.getResult() 332 333 change = None 334 if chdict: 335 wfd = defer.waitForDeferred(changes.Change.fromChdict(self.master, chdict)) 336 yield wfd 337 change = wfd.getResult() 338 339 # Define setid for this set of changed repositories 340 wfd = defer.waitForDeferred(self.master.db.sourcestampsets.addSourceStampSet()) 341 yield wfd 342 setid = wfd.getResult() 343 344 wfd = defer.waitForDeferred(self.master.db.sourcestamps.addSourceStamp( 345 branch=change.branch, 346 revision=change.revision, 347 repository=change.repository, 348 project=change.project, 349 changeids=changeids, 350 sourcestampsetid=setid)) 351 yield wfd 352 wfd.getResult() 353 354 wfd = defer.waitForDeferred(self.addBuildsetForSourceStamp( 355 setid=setid, reason=reason, 356 external_idstring=external_idstring, 357 builderNames=builderNames, 358 properties=properties)) 359 yield wfd 360 361 yield wfd.getResult()
362 363 @defer.deferredGenerator
364 - def addBuildsetForChangesMultiRepo(self, reason='', external_idstring=None, 365 changeids=[], builderNames=None, properties=None):
366 assert changeids is not [] 367 chDicts = {} 368 369 def getChange(changeid = None): 370 d = self.master.db.changes.getChange(changeid) 371 def chdict2change(chdict): 372 if not chdict: 373 return None 374 return changes.Change.fromChdict(self.master, chdict)
375 d.addCallback(chdict2change) 376 377 def groupChange(change): 378 if change.repository not in chDicts: 379 chDicts[change.repository] = [] 380 chDicts[change.repository].append(change) 381 382 def get_changeids_from_repo(repository): 383 changeids = [] 384 for change in chDicts[repository]: 385 changeids.append(change.number) 386 return changeids 387 388 def create_sourcestamp(changeids, change = None, setid = None): 389 def add_sourcestamp(setid, changeids = None): 390 return self.master.db.sourcestamps.addSourceStamp( 391 branch=change.branch, 392 revision=change.revision, 393 repository=change.repository, 394 project=change.project, 395 changeids=changeids, 396 setid=setid) 397 d.addCallback(add_sourcestamp, setid = setid, changeids = changeids) 398 return d 399 400 def create_sourcestamp_without_changes(setid, repository): 401 return self.master.db.sourcestamps.addSourceStamp( 402 branch=self.default_branch, 403 revision=None, 404 repository=repository, 405 project=self.default_project, 406 changeids=changeids, 407 sourcestampsetid=setid) 408 409 d = defer.Deferred() 410 if self.repositories is None: 411 # attributes for this sourcestamp will be based on the most recent 412 # change, so fetch the change with the highest id (= old single 413 # sourcestamp functionality) 414 d.addCallBack(getChange,changeid=max(changeids)) 415 d.addCallBack(groupChange) 416 else: 417 for changeid in changeids: 418 d.addCallBack(getChange,changeid = changeid) 419 d.addCallBack(groupChange) 420 421 # Define setid for this set of changed repositories 422 wfd = defer.waitForDeferred(self.master.db.sourcestampsets.addSourceStampSet) 423 yield wfd 424 setid = wfd.getResult() 425 426 #process all unchanged repositories 427 if self.repositories is not None: 428 for repo in self.repositories: 429 if repo not in chDicts: 430 # repository was not changed 431 # call create_sourcestamp 432 d.addCallback(create_sourcestamp_without_changes, setid, repo) 433 434 # process all changed 435 for repo in chDicts: 436 d.addCallback(get_changeids_from_repo, repository = repo) 437 d.addCallback(create_sourcestamp, setid = setid, change=chDicts[repo][-1]) 438 439 # add one buildset, this buildset is connected to the sourcestamps by the setid 440 d.addCallback(self.addBuildsetForSourceStamp, setid=setid, reason=reason, 441 external_idstring=external_idstring, 442 builderNames=builderNames, 443 properties=properties) 444 445 yield d 446 447 448 @defer.deferredGenerator
449 - def addBuildsetForSourceStamp(self, ssid=None, setid=None, reason='', external_idstring=None, 450 properties=None, builderNames=None):
451 """ 452 Add a buildset for the given, already-existing sourcestamp. 453 454 This method will add any properties provided to the scheduler 455 constructor to the buildset, and will call the master's 456 L{BuildMaster.addBuildset} method with the appropriate parameters, and 457 return the same result. 458 459 @param reason: reason for this buildset 460 @type reason: unicode string 461 @param external_idstring: external identifier for this buildset, or None 462 @param properties: a properties object containing initial properties for 463 the buildset 464 @type properties: L{buildbot.process.properties.Properties} 465 @param builderNames: builders to name in the buildset (defaults to 466 C{self.builderNames}) 467 @param setid: idenitification of a set of sourcestamps 468 @returns: (buildset ID, buildrequest IDs) via Deferred 469 """ 470 assert (ssid is None and setid is not None) \ 471 or (ssid is not None and setid is None), "pass a single sourcestamp OR set not both" 472 473 # combine properties 474 if properties: 475 properties.updateFromProperties(self.properties) 476 else: 477 properties = self.properties 478 479 # apply the default builderNames 480 if not builderNames: 481 builderNames = self.builderNames 482 483 # translate properties object into a dict as required by the 484 # addBuildset method 485 properties_dict = properties.asDict() 486 487 if setid == None: 488 if ssid != None: 489 wfd = defer.waitForDeferred(self.master.db.sourcestamps.getSourceStamp(ssid)) 490 yield wfd 491 ssdict = wfd.getResult() 492 setid = ssdict['sourcestampsetid'] 493 else: 494 # no sourcestamp and no sets 495 yield None 496 497 wfd = defer.waitForDeferred(self.master.addBuildset( 498 sourcestampsetid=setid, reason=reason, 499 properties=properties_dict, 500 builderNames=builderNames, 501 external_idstring=external_idstring)) 502 yield wfd 503 yield wfd.getResult()
504