1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 from zope.interface import implements
17 from twisted.python import failure, log
18 from twisted.application import service
19 from twisted.internet import defer
20 from buildbot.process.properties import Properties
21 from buildbot.util import ComparableMixin
22 from buildbot.changes import changes
23 from buildbot import config, interfaces
26 """
27 Base class for all schedulers; this provides the equipment to manage
28 reconfigurations and to handle basic scheduler state. It also provides
29 utility methods to begin various sorts of builds.
30
31 Subclasses should add any configuration-derived attributes to
32 C{base.Scheduler.compare_attrs}.
33 """
34
35 implements(interfaces.IScheduler)
36
37 compare_attrs = ('name', 'builderNames', 'properties')
38
39 - def __init__(self, name, builderNames, properties):
40 """
41 Initialize a Scheduler.
42
43 @param name: name of this scheduler (used as a key for state)
44 @type name: unicode
45
46 @param builderNames: list of builders this scheduler may start
47 @type builderNames: list of unicode
48
49 @param properties: properties to add to builds triggered by this
50 scheduler
51 @type properties: dictionary
52
53 @param consumeChanges: true if this scheduler wishes to be informed
54 about the addition of new changes. Defaults to False. This should
55 be passed explicitly from subclasses to indicate their interest in
56 consuming changes.
57 @type consumeChanges: boolean
58 """
59 service.MultiService.__init__(self)
60 self.name = name
61 "name of this scheduler; used to identify replacements on reconfig"
62
63 ok = True
64 if not isinstance(builderNames, (list, tuple)):
65 ok = False
66 else:
67 for b in builderNames:
68 if not isinstance(b, basestring):
69 ok = False
70 if not ok:
71 config.error(
72 "The builderNames argument to a scheduler must be a list "
73 "of Builder names.")
74
75 self.builderNames = builderNames
76 "list of builder names to start in each buildset"
77
78 self.properties = Properties()
79 "properties that are contributed to each buildset"
80 self.properties.update(properties, "Scheduler")
81 self.properties.setProperty("scheduler", name, "Scheduler")
82
83 self.objectid = None
84
85 self.master = None
86
87
88 self._change_subscription = None
89 self._change_consumption_lock = defer.DeferredLock()
90 self._objectid = None
91
92
93
96
99
101 d = defer.maybeDeferred(self._stopConsumingChanges)
102 d.addCallback(lambda _ : service.MultiService.stopService(self))
103 return d
104
105
106
107 @defer.deferredGenerator
109 """
110 For use by subclasses; get a named state value from the scheduler's
111 state, defaulting to DEFAULT.
112
113 @param name: name of the value to retrieve
114 @param default: (optional) value to return if C{name} is not present
115 @returns: state value via a Deferred
116 @raises KeyError: if C{name} is not present and no default is given
117 @raises TypeError: if JSON parsing fails
118 """
119
120 if self._objectid is None:
121 wfd = defer.waitForDeferred(
122 self.master.db.state.getObjectId(self.name,
123 self.__class__.__name__))
124 yield wfd
125 self._objectid = wfd.getResult()
126
127 wfd = defer.waitForDeferred(
128 self.master.db.state.getState(self._objectid, *args, **kwargs))
129 yield wfd
130 yield wfd.getResult()
131
132 @defer.deferredGenerator
134 """
135 For use by subclasses; set a named state value in the scheduler's
136 persistent state. Note that value must be json-able.
137
138 @param name: the name of the value to change
139 @param value: the value to set - must be a JSONable object
140 @param returns: Deferred
141 @raises TypeError: if JSONification fails
142 """
143
144 if self._objectid is None:
145 wfd = defer.waitForDeferred(
146 self.master.db.state.getObjectId(self.name,
147 self.__class__.__name__))
148 yield wfd
149 self._objectid = wfd.getResult()
150
151 wfd = defer.waitForDeferred(
152 self.master.db.state.setState(self._objectid, key, value))
153 yield wfd
154 wfd.getResult()
155
156
157
158
159
161 "Returns the list of builder names"
162 return self.builderNames
163
165 "Returns a list of the next times that builds are scheduled, if known."
166 return []
167
168
169
172 """
173 Subclasses should call this method from startService to register to
174 receive changes. The BaseScheduler class will take care of filtering
175 the changes (using change_filter) and (if fileIsImportant is not None)
176 classifying them. See L{gotChange}. Returns a Deferred.
177
178 @param fileIsImportant: a callable provided by the user to distinguish
179 important and unimportant changes
180 @type fileIsImportant: callable
181
182 @param change_filter: a filter to determine which changes are even
183 considered by this scheduler, or C{None} to consider all changes
184 @type change_filter: L{buildbot.changes.filter.ChangeFilter} instance
185
186 @param onlyImportant: If True, only important changes, as specified by
187 fileIsImportant, will be added to the buildset.
188 @type onlyImportant: boolean
189
190 """
191 assert fileIsImportant is None or callable(fileIsImportant)
192
193
194 assert not self._change_subscription
195 def changeCallback(change):
196
197 if not self._change_subscription:
198 return
199
200 if change_filter and not change_filter.filter_change(change):
201 return
202 if fileIsImportant:
203 try:
204 important = fileIsImportant(change)
205 if not important and onlyImportant:
206 return
207 except:
208 log.err(failure.Failure(),
209 'in fileIsImportant check for %s' % change)
210 return
211 else:
212 important = True
213
214
215
216 d = self._change_consumption_lock.acquire()
217 d.addCallback(lambda _ : self.gotChange(change, important))
218 def release(x):
219 self._change_consumption_lock.release()
220 d.addBoth(release)
221 d.addErrback(log.err, 'while processing change')
222 self._change_subscription = self.master.subscribeToChanges(changeCallback)
223
224 return defer.succeed(None)
225
227
228
229
230
231 d = self._change_consumption_lock.acquire()
232 def stop(x):
233 if self._change_subscription:
234 self._change_subscription.unsubscribe()
235 self._change_subscription = None
236 self._change_consumption_lock.release()
237 d.addBoth(stop)
238 return d
239
241 """
242 Called when a change is received; returns a Deferred. If the
243 C{fileIsImportant} parameter to C{startConsumingChanges} was C{None},
244 then all changes are considered important.
245
246 @param change: the new change object
247 @type change: L{buildbot.changes.changes.Change} instance
248 @param important: true if this is an important change, according to
249 C{fileIsImportant}.
250 @type important: boolean
251 @returns: Deferred
252 """
253 raise NotImplementedError
254
255
256
257 @defer.deferredGenerator
258 - def addBuildsetForLatest(self, reason='', external_idstring=None,
259 branch=None, repository='', project='',
260 builderNames=None, properties=None):
261 """
262 Add a buildset for the 'latest' source in the given branch,
263 repository, and project. This will create a relative sourcestamp for
264 the buildset.
265
266 This method will add any properties provided to the scheduler
267 constructor to the buildset, and will call the master's addBuildset
268 method with the appropriate parameters.
269
270 @param reason: reason for this buildset
271 @type reason: unicode string
272 @param external_idstring: external identifier for this buildset, or None
273 @param branch: branch to build (note that None often has a special meaning)
274 @param repository: repository name for sourcestamp
275 @param project: project name for sourcestamp
276 @param builderNames: builders to name in the buildset (defaults to
277 C{self.builderNames})
278 @param properties: a properties object containing initial properties for
279 the buildset
280 @type properties: L{buildbot.process.properties.Properties}
281 @returns: (buildset ID, buildrequest IDs) via Deferred
282 """
283
284 wfd = defer.waitForDeferred(self.master.db.sourcestampsets.addSourceStampSet())
285 yield wfd
286 setid = wfd.getResult()
287
288 wfd = defer.waitForDeferred(self.master.db.sourcestamps.addSourceStamp(
289 branch=branch, revision=None, repository=repository,
290 project=project, sourcestampsetid=setid))
291 yield wfd
292 wfd.getResult()
293
294 wfd = defer.waitForDeferred(self.addBuildsetForSourceStamp(
295 setid=setid, reason=reason,
296 external_idstring=external_idstring,
297 builderNames=builderNames,
298 properties=properties))
299 yield wfd
300 yield wfd.getResult()
301
302 @defer.deferredGenerator
303 - def addBuildsetForChanges(self, reason='', external_idstring=None,
304 changeids=[], builderNames=None, properties=None):
305 """
306 Add a buildset for the combination of the given changesets, creating
307 a sourcestamp based on those changes. The sourcestamp for the buildset
308 will reference all of the indicated changes.
309
310 This method will add any properties provided to the scheduler
311 constructor to the buildset, and will call the master's addBuildset
312 method with the appropriate parameters.
313
314 @param reason: reason for this buildset
315 @type reason: unicode string
316 @param external_idstring: external identifier for this buildset, or None
317 @param changeids: nonempty list of changes to include in this buildset
318 @param builderNames: builders to name in the buildset (defaults to
319 C{self.builderNames})
320 @param properties: a properties object containing initial properties for
321 the buildset
322 @type properties: L{buildbot.process.properties.Properties}
323 @returns: (buildset ID, buildrequest IDs) via Deferred
324 """
325 assert changeids is not []
326
327
328
329 wfd = defer.waitForDeferred(self.master.db.changes.getChange(max(changeids)))
330 yield wfd
331 chdict = wfd.getResult()
332
333 change = None
334 if chdict:
335 wfd = defer.waitForDeferred(changes.Change.fromChdict(self.master, chdict))
336 yield wfd
337 change = wfd.getResult()
338
339
340 wfd = defer.waitForDeferred(self.master.db.sourcestampsets.addSourceStampSet())
341 yield wfd
342 setid = wfd.getResult()
343
344 wfd = defer.waitForDeferred(self.master.db.sourcestamps.addSourceStamp(
345 branch=change.branch,
346 revision=change.revision,
347 repository=change.repository,
348 project=change.project,
349 changeids=changeids,
350 sourcestampsetid=setid))
351 yield wfd
352 wfd.getResult()
353
354 wfd = defer.waitForDeferred(self.addBuildsetForSourceStamp(
355 setid=setid, reason=reason,
356 external_idstring=external_idstring,
357 builderNames=builderNames,
358 properties=properties))
359 yield wfd
360
361 yield wfd.getResult()
362
363 @defer.deferredGenerator
375 d.addCallback(chdict2change)
376
377 def groupChange(change):
378 if change.repository not in chDicts:
379 chDicts[change.repository] = []
380 chDicts[change.repository].append(change)
381
382 def get_changeids_from_repo(repository):
383 changeids = []
384 for change in chDicts[repository]:
385 changeids.append(change.number)
386 return changeids
387
388 def create_sourcestamp(changeids, change = None, setid = None):
389 def add_sourcestamp(setid, changeids = None):
390 return self.master.db.sourcestamps.addSourceStamp(
391 branch=change.branch,
392 revision=change.revision,
393 repository=change.repository,
394 project=change.project,
395 changeids=changeids,
396 setid=setid)
397 d.addCallback(add_sourcestamp, setid = setid, changeids = changeids)
398 return d
399
400 def create_sourcestamp_without_changes(setid, repository):
401 return self.master.db.sourcestamps.addSourceStamp(
402 branch=self.default_branch,
403 revision=None,
404 repository=repository,
405 project=self.default_project,
406 changeids=changeids,
407 sourcestampsetid=setid)
408
409 d = defer.Deferred()
410 if self.repositories is None:
411
412
413
414 d.addCallBack(getChange,changeid=max(changeids))
415 d.addCallBack(groupChange)
416 else:
417 for changeid in changeids:
418 d.addCallBack(getChange,changeid = changeid)
419 d.addCallBack(groupChange)
420
421
422 wfd = defer.waitForDeferred(self.master.db.sourcestampsets.addSourceStampSet)
423 yield wfd
424 setid = wfd.getResult()
425
426
427 if self.repositories is not None:
428 for repo in self.repositories:
429 if repo not in chDicts:
430
431
432 d.addCallback(create_sourcestamp_without_changes, setid, repo)
433
434
435 for repo in chDicts:
436 d.addCallback(get_changeids_from_repo, repository = repo)
437 d.addCallback(create_sourcestamp, setid = setid, change=chDicts[repo][-1])
438
439
440 d.addCallback(self.addBuildsetForSourceStamp, setid=setid, reason=reason,
441 external_idstring=external_idstring,
442 builderNames=builderNames,
443 properties=properties)
444
445 yield d
446
447
448 @defer.deferredGenerator
449 - def addBuildsetForSourceStamp(self, ssid=None, setid=None, reason='', external_idstring=None,
450 properties=None, builderNames=None):
451 """
452 Add a buildset for the given, already-existing sourcestamp.
453
454 This method will add any properties provided to the scheduler
455 constructor to the buildset, and will call the master's
456 L{BuildMaster.addBuildset} method with the appropriate parameters, and
457 return the same result.
458
459 @param reason: reason for this buildset
460 @type reason: unicode string
461 @param external_idstring: external identifier for this buildset, or None
462 @param properties: a properties object containing initial properties for
463 the buildset
464 @type properties: L{buildbot.process.properties.Properties}
465 @param builderNames: builders to name in the buildset (defaults to
466 C{self.builderNames})
467 @param setid: idenitification of a set of sourcestamps
468 @returns: (buildset ID, buildrequest IDs) via Deferred
469 """
470 assert (ssid is None and setid is not None) \
471 or (ssid is not None and setid is None), "pass a single sourcestamp OR set not both"
472
473
474 if properties:
475 properties.updateFromProperties(self.properties)
476 else:
477 properties = self.properties
478
479
480 if not builderNames:
481 builderNames = self.builderNames
482
483
484
485 properties_dict = properties.asDict()
486
487 if setid == None:
488 if ssid != None:
489 wfd = defer.waitForDeferred(self.master.db.sourcestamps.getSourceStamp(ssid))
490 yield wfd
491 ssdict = wfd.getResult()
492 setid = ssdict['sourcestampsetid']
493 else:
494
495 yield None
496
497 wfd = defer.waitForDeferred(self.master.addBuildset(
498 sourcestampsetid=setid, reason=reason,
499 properties=properties_dict,
500 builderNames=builderNames,
501 external_idstring=external_idstring))
502 yield wfd
503 yield wfd.getResult()
504