buildbot.schedulers.timed

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 import time 17 from buildbot import util 18 from buildbot.schedulers import base 19 from twisted.internet import defer, reactor 20 from twisted.python import log 21 from buildbot.changes import filter

22 23 -class Timed(base.BaseScheduler):

24 """ 25 Parent class for timed schedulers. This takes care of the (surprisingly 26 subtle) mechanics of ensuring that each timed actuation runs to completion 27 before the service stops. 28 """ 29 30 compare_attrs = base.BaseScheduler.compare_attrs 31

32 - def __init__(self, name, builderNames, properties={}):

33 base.BaseScheduler.__init__(self, name, builderNames, properties) 34 35 # tracking for when to start the next build 36 self.lastActuated = None 37 38 # A lock to make sure that each actuation occurs without interruption. 39 # This lock governs actuateAt, actuateAtTimer, and actuateOk 40 self.actuationLock = defer.DeferredLock() 41 self.actuateOk = False 42 self.actuateAt = None 43 self.actuateAtTimer = None 44 45 self._reactor = reactor # patched by tests

46

47 - def startService(self):

48 base.BaseScheduler.startService(self) 49 50 # no need to lock this; nothing else can run before the service is started 51 self.actuateOk = True 52 53 # get the scheduler's last_build time (note: only done at startup) 54 d = self.getState('last_build', None) 55 def set_last(lastActuated): 56 self.lastActuated = lastActuated

57 d.addCallback(set_last) 58 59 # schedule the next build 60 d.addCallback(lambda _ : self.scheduleNextBuild()) 61 62 # give subclasses a chance to start up 63 d.addCallback(lambda _ : self.startTimedSchedulerService()) 64 65 # startService does not return a Deferred, so handle errors with a traceback 66 d.addErrback(log.err, "while initializing %s '%s'" % 67 (self.__class__.__name__, self.name))

68

69 - def startTimedSchedulerService(self):

70 """Hook for subclasses to participate in the L{startService} process; 71 can return a Deferred"""

72

73 - def stopService(self):

74 # shut down any pending actuation, and ensure that we wait for any 75 # current actuation to complete by acquiring the lock. This ensures 76 # that no build will be scheduled after stopService is complete. 77 d = self.actuationLock.acquire() 78 def stop_actuating(_): 79 self.actuateOk = False 80 self.actuateAt = None 81 if self.actuateAtTimer: 82 self.actuateAtTimer.cancel() 83 self.actuateAtTimer = None

84 d.addCallback(stop_actuating) 85 d.addCallback(lambda _ : self.actuationLock.release()) 86 87 # and chain to the parent class 88 d.addCallback(lambda _ : base.BaseScheduler.stopService(self)) 89 return d 90 91 ## Scheduler methods 92

93 - def getPendingBuildTimes(self):

94 # take the latest-calculated value of actuateAt as a reasonable 95 # estimate 96 return [ self.actuateAt ]

97 98 ## Timed methods 99

100 - def startBuild(self):

101 """The time has come to start a new build. Returns a Deferred. 102 Override in subclasses.""" 103 raise NotImplementedError

104

105 - def getNextBuildTime(self, lastActuation):

106 """ 107 Called by to calculate the next time to actuate a BuildSet. Override 108 in subclasses. To trigger a fresh call to this method, use 109 L{rescheduleNextBuild}. 110 111 @param lastActuation: the time of the last actuation, or None for never 112 113 @returns: a Deferred firing with the next time a build should occur (in 114 the future), or None for never. 115 """ 116 raise NotImplementedError

117

118 - def scheduleNextBuild(self):

119 """ 120 Schedule the next build, re-invoking L{getNextBuildTime}. This can be 121 called at any time, and it will avoid contention with builds being 122 started concurrently. 123 124 @returns: Deferred 125 """ 126 d = self.actuationLock.acquire() 127 d.addCallback(lambda _ : self._scheduleNextBuild_locked()) 128 # always release the lock 129 def release(x): 130 self.actuationLock.release() 131 return x

132 d.addBoth(release) 133 return d 134 135 ## utilities 136

137 - def now(self):

138 "Similar to util.now, but patchable by tests" 139 return util.now(self._reactor)

140

141 - def _scheduleNextBuild_locked(self):

142 # clear out the existing timer 143 if self.actuateAtTimer: 144 self.actuateAtTimer.cancel() 145 self.actuateAtTimer = None 146 147 # calculate the new time 148 d = self.getNextBuildTime(self.lastActuated) 149 150 # set up the new timer 151 def set_timer(actuateAt): 152 now = self.now() 153 self.actuateAt = max(actuateAt, now) 154 if actuateAt is not None: 155 untilNext = self.actuateAt - now 156 if untilNext == 0: 157 log.msg(("%s: missed scheduled build time, so building " 158 "immediately") % self.name) 159 self.actuateAtTimer = self._reactor.callLater(untilNext, 160 self._actuate)

161 d.addCallback(set_timer) 162 163 return d 164

165 - def _actuate(self):

166 # called from the timer when it's time to start a build 167 self.actuateAtTimer = None 168 self.lastActuated = self.actuateAt 169 170 d = self.actuationLock.acquire() 171 172 @defer.deferredGenerator 173 def set_state_and_start(_): 174 # bail out if we shouldn't be actuating anymore 175 if not self.actuateOk: 176 return 177 178 # mark the last build time 179 self.actuateAt = None 180 wfd = defer.waitForDeferred(self.setState('last_build', 181 self.lastActuated)) 182 yield wfd 183 wfd.getResult() 184 185 # start the build 186 wfd = defer.waitForDeferred(self.startBuild()) 187 yield wfd 188 wfd.getResult() 189 190 # schedule the next build (noting the lock is already held) 191 wfd = defer.waitForDeferred(self._scheduleNextBuild_locked()) 192 yield wfd 193 wfd.getResult()

194 d.addCallback(set_state_and_start) 195 196 def unlock(x): 197 self.actuationLock.release() 198 return x 199 d.addBoth(unlock) 200 201 # this function can't return a deferred, so handle any failures via 202 # log.err 203 d.addErrback(log.err, 'while actuating') 204

205 206 -class Periodic(Timed):

207 compare_attrs = Timed.compare_attrs + ('periodicBuildTimer', 'branch',) 208

209 - def __init__(self, name, builderNames, periodicBuildTimer, 210 branch=None, properties={}, onlyImportant=False):

211 Timed.__init__(self, name=name, builderNames=builderNames, 212 properties=properties) 213 assert periodicBuildTimer > 0, "periodicBuildTimer must be positive" 214 self.periodicBuildTimer = periodicBuildTimer 215 self.branch = branch 216 self.reason = "The Periodic scheduler named '%s' triggered this build" % self.name

217

218 - def getNextBuildTime(self, lastActuated):

219 if lastActuated is None: 220 return defer.succeed(self.now()) # meaning "ASAP" 221 else: 222 return defer.succeed(lastActuated + self.periodicBuildTimer)

223

224 - def startBuild(self):

225 return self.addBuildsetForLatest(reason=self.reason, branch=self.branch)

226

227 -class Nightly(Timed):

228 compare_attrs = (Timed.compare_attrs 229 + ('minute', 'hour', 'dayOfMonth', 'month', 230 'dayOfWeek', 'onlyIfChanged', 'fileIsImportant', 231 'change_filter', 'onlyImportant',)) 232

233 - class NoBranch: pass

234 - def __init__(self, name, builderNames, minute=0, hour='*', 235 dayOfMonth='*', month='*', dayOfWeek='*', 236 branch=NoBranch, fileIsImportant=None, onlyIfChanged=False, 237 properties={}, change_filter=None, onlyImportant=False):

238 Timed.__init__(self, name=name, builderNames=builderNames, properties=properties) 239 240 # If True, only important changes will be added to the buildset. 241 self.onlyImportant = onlyImportant 242 243 if fileIsImportant: 244 assert callable(fileIsImportant), \ 245 "fileIsImportant must be a callable" 246 assert branch is not Nightly.NoBranch, \ 247 "Nightly parameter 'branch' is required" 248 249 self.minute = minute 250 self.hour = hour 251 self.dayOfMonth = dayOfMonth 252 self.month = month 253 self.dayOfWeek = dayOfWeek 254 self.branch = branch 255 self.onlyIfChanged = onlyIfChanged 256 self.fileIsImportant = fileIsImportant 257 self.change_filter = filter.ChangeFilter.fromSchedulerConstructorArgs( 258 change_filter=change_filter) 259 self.reason = "The Nightly scheduler named '%s' triggered this build" % self.name

260

261 - def startTimedSchedulerService(self):

262 if self.onlyIfChanged: 263 return self.startConsumingChanges(fileIsImportant=self.fileIsImportant, 264 change_filter=self.change_filter, 265 onlyImportant=self.onlyImportant) 266 else: 267 return self.master.db.schedulers.flushChangeClassifications(self.schedulerid)

268

269 - def gotChange(self, change, important):

270 # both important and unimportant changes on our branch are recorded, as 271 # we will include all such changes in any buildsets we start. Note 272 # that we must check the branch here because it is not included in the 273 # change filter 274 if change.branch != self.branch: 275 return defer.succeed(None) # don't care about this change 276 return self.master.db.schedulers.classifyChanges( 277 self.schedulerid, { change.number : important })

278

279 - def getNextBuildTime(self, lastActuated):

280 def addTime(timetuple, secs): 281 return time.localtime(time.mktime(timetuple)+secs)

282 283 def check(ourvalue, value): 284 if ourvalue == '*': return True 285 if isinstance(ourvalue, int): return value == ourvalue 286 return (value in ourvalue)

287 288 dateTime = time.localtime(lastActuated or self.now()) 289 290 # Remove seconds by advancing to at least the next minute 291 dateTime = addTime(dateTime, 60-dateTime[5]) 292 293 # Now we just keep adding minutes until we find something that matches 294 # TODO: use a smarter algorithm, now that we have thorough tests 295 296 yearLimit = dateTime[0]+2 # only check 2 years (a lot of minutes!) 297 def isRunTime(timetuple): 298 299 if not check(self.minute, timetuple[4]): 300 return False 301 302 if not check(self.hour, timetuple[3]): 303 return False 304 305 if not check(self.month, timetuple[1]): 306 return False 307 308 if self.dayOfMonth != '*' and self.dayOfWeek != '*': 309 # They specified both day(s) of month AND day(s) of week. 310 # This means that we only have to match one of the two. If 311 # neither one matches, this time is not the right time. 312 if not (check(self.dayOfMonth, timetuple[2]) or 313 check(self.dayOfWeek, timetuple[6])): 314 return False 315 else: 316 if not check(self.dayOfMonth, timetuple[2]): 317 return False 318 319 if not check(self.dayOfWeek, timetuple[6]): 320 return False 321 322 return True 323 324 while not isRunTime(dateTime): 325 dateTime = addTime(dateTime, 60) 326 assert dateTime[0] < yearLimit, 'Something is wrong with this code' 327 return defer.succeed(time.mktime(dateTime)) 328 329 @defer.deferredGenerator

330 - def startBuild(self):

331 scheds = self.master.db.schedulers 332 # if onlyIfChanged is True, then we will skip this build if no 333 # important changes have occurred since the last invocation 334 if self.onlyIfChanged: 335 wfd = defer.waitForDeferred(scheds.getChangeClassifications(self.schedulerid)) 336 yield wfd 337 classifications = wfd.getResult() 338 339 # see if we have any important changes 340 for imp in classifications.itervalues(): 341 if imp: 342 break 343 else: 344 log.msg(("Nightly Scheduler <%s>: skipping build " + 345 "- No important changes on configured branch") % self.name) 346 return 347 348 changeids = sorted(classifications.keys()) 349 wfd = defer.waitForDeferred( 350 self.addBuildsetForChanges(reason=self.reason, changeids=changeids)) 351 yield wfd 352 wfd.getResult() 353 354 max_changeid = changeids[-1] # (changeids are sorted) 355 wfd = defer.waitForDeferred( 356 scheds.flushChangeClassifications(self.schedulerid, 357 less_than=max_changeid+1)) 358 yield wfd 359 wfd.getResult() 360 else: 361 # start a build of the latest revision, whatever that is 362 wfd = defer.waitForDeferred( 363 self.addBuildsetForLatest(reason=self.reason, branch=self.branch)) 364 yield wfd 365 wfd.getResult()

366

Source Code for Module buildbot.schedulers.timed