buildbot.schedulers.timed

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 import time 17 from buildbot import util 18 from buildbot.schedulers import base 19 from twisted.internet import defer, reactor 20 from twisted.python import log 21 from buildbot import config 22 from buildbot.changes import filter

23 24 -class Timed(base.BaseScheduler):

25 """ 26 Parent class for timed schedulers. This takes care of the (surprisingly 27 subtle) mechanics of ensuring that each timed actuation runs to completion 28 before the service stops. 29 """ 30 31 compare_attrs = base.BaseScheduler.compare_attrs 32

33 - def __init__(self, name, builderNames, properties={}):

34 base.BaseScheduler.__init__(self, name, builderNames, properties) 35 36 # tracking for when to start the next build 37 self.lastActuated = None 38 39 # A lock to make sure that each actuation occurs without interruption. 40 # This lock governs actuateAt, actuateAtTimer, and actuateOk 41 self.actuationLock = defer.DeferredLock() 42 self.actuateOk = False 43 self.actuateAt = None 44 self.actuateAtTimer = None 45 46 self._reactor = reactor # patched by tests

47

48 - def startService(self):

49 base.BaseScheduler.startService(self) 50 51 # no need to lock this; nothing else can run before the service is started 52 self.actuateOk = True 53 54 # get the scheduler's last_build time (note: only done at startup) 55 d = self.getState('last_build', None) 56 def set_last(lastActuated): 57 self.lastActuated = lastActuated

58 d.addCallback(set_last) 59 60 # schedule the next build 61 d.addCallback(lambda _ : self.scheduleNextBuild()) 62 63 # give subclasses a chance to start up 64 d.addCallback(lambda _ : self.startTimedSchedulerService()) 65 66 # startService does not return a Deferred, so handle errors with a traceback 67 d.addErrback(log.err, "while initializing %s '%s'" % 68 (self.__class__.__name__, self.name))

69

70 - def startTimedSchedulerService(self):

71 """Hook for subclasses to participate in the L{startService} process; 72 can return a Deferred"""

73

74 - def stopService(self):

75 # shut down any pending actuation, and ensure that we wait for any 76 # current actuation to complete by acquiring the lock. This ensures 77 # that no build will be scheduled after stopService is complete. 78 d = self.actuationLock.acquire() 79 def stop_actuating(_): 80 self.actuateOk = False 81 self.actuateAt = None 82 if self.actuateAtTimer: 83 self.actuateAtTimer.cancel() 84 self.actuateAtTimer = None

85 d.addCallback(stop_actuating) 86 d.addCallback(lambda _ : self.actuationLock.release()) 87 88 # and chain to the parent class 89 d.addCallback(lambda _ : base.BaseScheduler.stopService(self)) 90 return d 91 92 ## Scheduler methods 93

94 - def getPendingBuildTimes(self):

95 # take the latest-calculated value of actuateAt as a reasonable 96 # estimate 97 return [ self.actuateAt ]

98 99 ## Timed methods 100

101 - def startBuild(self):

102 """The time has come to start a new build. Returns a Deferred. 103 Override in subclasses.""" 104 raise NotImplementedError

105

106 - def getNextBuildTime(self, lastActuation):

107 """ 108 Called by to calculate the next time to actuate a BuildSet. Override 109 in subclasses. To trigger a fresh call to this method, use 110 L{rescheduleNextBuild}. 111 112 @param lastActuation: the time of the last actuation, or None for never 113 114 @returns: a Deferred firing with the next time a build should occur (in 115 the future), or None for never. 116 """ 117 raise NotImplementedError

118

119 - def scheduleNextBuild(self):

120 """ 121 Schedule the next build, re-invoking L{getNextBuildTime}. This can be 122 called at any time, and it will avoid contention with builds being 123 started concurrently. 124 125 @returns: Deferred 126 """ 127 d = self.actuationLock.acquire() 128 d.addCallback(lambda _ : self._scheduleNextBuild_locked()) 129 # always release the lock 130 def release(x): 131 self.actuationLock.release() 132 return x

133 d.addBoth(release) 134 return d 135 136 ## utilities 137

138 - def now(self):

139 "Similar to util.now, but patchable by tests" 140 return util.now(self._reactor)

141

142 - def _scheduleNextBuild_locked(self):

143 # clear out the existing timer 144 if self.actuateAtTimer: 145 self.actuateAtTimer.cancel() 146 self.actuateAtTimer = None 147 148 # calculate the new time 149 d = self.getNextBuildTime(self.lastActuated) 150 151 # set up the new timer 152 def set_timer(actuateAt): 153 now = self.now() 154 self.actuateAt = max(actuateAt, now) 155 if actuateAt is not None: 156 untilNext = self.actuateAt - now 157 if untilNext == 0: 158 log.msg(("%s: missed scheduled build time, so building " 159 "immediately") % self.name) 160 self.actuateAtTimer = self._reactor.callLater(untilNext, 161 self._actuate)

162 d.addCallback(set_timer) 163 164 return d 165

166 - def _actuate(self):

167 # called from the timer when it's time to start a build 168 self.actuateAtTimer = None 169 self.lastActuated = self.actuateAt 170 171 d = self.actuationLock.acquire() 172 173 @defer.deferredGenerator 174 def set_state_and_start(_): 175 # bail out if we shouldn't be actuating anymore 176 if not self.actuateOk: 177 return 178 179 # mark the last build time 180 self.actuateAt = None 181 wfd = defer.waitForDeferred(self.setState('last_build', 182 self.lastActuated)) 183 yield wfd 184 wfd.getResult() 185 186 # start the build 187 wfd = defer.waitForDeferred(self.startBuild()) 188 yield wfd 189 wfd.getResult() 190 191 # schedule the next build (noting the lock is already held) 192 wfd = defer.waitForDeferred(self._scheduleNextBuild_locked()) 193 yield wfd 194 wfd.getResult()

195 d.addCallback(set_state_and_start) 196 197 def unlock(x): 198 self.actuationLock.release() 199 return x 200 d.addBoth(unlock) 201 202 # this function can't return a deferred, so handle any failures via 203 # log.err 204 d.addErrback(log.err, 'while actuating') 205

206 207 -class Periodic(Timed):

208 compare_attrs = Timed.compare_attrs + ('periodicBuildTimer', 'branch',) 209

210 - def __init__(self, name, builderNames, periodicBuildTimer, 211 branch=None, properties={}, onlyImportant=False):

212 Timed.__init__(self, name=name, builderNames=builderNames, 213 properties=properties) 214 if periodicBuildTimer <= 0: 215 config.error( 216 "periodicBuildTimer must be positive") 217 self.periodicBuildTimer = periodicBuildTimer 218 self.branch = branch 219 self.reason = "The Periodic scheduler named '%s' triggered this build" % self.name

220

221 - def getNextBuildTime(self, lastActuated):

222 if lastActuated is None: 223 return defer.succeed(self.now()) # meaning "ASAP" 224 else: 225 return defer.succeed(lastActuated + self.periodicBuildTimer)

226

227 - def startBuild(self):

228 return self.addBuildsetForLatest(reason=self.reason, branch=self.branch)

229

230 -class Nightly(Timed):

231 compare_attrs = (Timed.compare_attrs 232 + ('minute', 'hour', 'dayOfMonth', 'month', 233 'dayOfWeek', 'onlyIfChanged', 'fileIsImportant', 234 'change_filter', 'onlyImportant', 'branch')) 235

236 - class NoBranch: pass

237 - def __init__(self, name, builderNames, minute=0, hour='*', 238 dayOfMonth='*', month='*', dayOfWeek='*', 239 branch=NoBranch, fileIsImportant=None, onlyIfChanged=False, 240 properties={}, change_filter=None, onlyImportant=False):

241 Timed.__init__(self, name=name, builderNames=builderNames, properties=properties) 242 243 # If True, only important changes will be added to the buildset. 244 self.onlyImportant = onlyImportant 245 246 if fileIsImportant and not callable(fileIsImportant): 247 config.error( 248 "fileIsImportant must be a callable") 249 if branch is Nightly.NoBranch: 250 config.error( 251 "Nightly parameter 'branch' is required") 252 253 self.minute = minute 254 self.hour = hour 255 self.dayOfMonth = dayOfMonth 256 self.month = month 257 self.dayOfWeek = dayOfWeek 258 self.branch = branch 259 self.onlyIfChanged = onlyIfChanged 260 self.fileIsImportant = fileIsImportant 261 self.change_filter = filter.ChangeFilter.fromSchedulerConstructorArgs( 262 change_filter=change_filter) 263 self.reason = "The Nightly scheduler named '%s' triggered this build" % self.name

264

265 - def startTimedSchedulerService(self):

266 if self.onlyIfChanged: 267 return self.startConsumingChanges(fileIsImportant=self.fileIsImportant, 268 change_filter=self.change_filter, 269 onlyImportant=self.onlyImportant) 270 else: 271 return self.master.db.schedulers.flushChangeClassifications(self.objectid)

272

273 - def gotChange(self, change, important):

274 # both important and unimportant changes on our branch are recorded, as 275 # we will include all such changes in any buildsets we start. Note 276 # that we must check the branch here because it is not included in the 277 # change filter 278 if change.branch != self.branch: 279 return defer.succeed(None) # don't care about this change 280 return self.master.db.schedulers.classifyChanges( 281 self.objectid, { change.number : important })

282

283 - def getNextBuildTime(self, lastActuated):

284 def addTime(timetuple, secs): 285 return time.localtime(time.mktime(timetuple)+secs)

286 287 def check(ourvalue, value): 288 if ourvalue == '*': return True 289 if isinstance(ourvalue, int): return value == ourvalue 290 return (value in ourvalue)

291 292 dateTime = time.localtime(lastActuated or self.now()) 293 294 # Remove seconds by advancing to at least the next minute 295 dateTime = addTime(dateTime, 60-dateTime[5]) 296 297 # Now we just keep adding minutes until we find something that matches 298 # TODO: use a smarter algorithm, now that we have thorough tests 299 300 yearLimit = dateTime[0]+2 # only check 2 years (a lot of minutes!) 301 def isRunTime(timetuple): 302 303 if not check(self.minute, timetuple[4]): 304 return False 305 306 if not check(self.hour, timetuple[3]): 307 return False 308 309 if not check(self.month, timetuple[1]): 310 return False 311 312 if self.dayOfMonth != '*' and self.dayOfWeek != '*': 313 # They specified both day(s) of month AND day(s) of week. 314 # This means that we only have to match one of the two. If 315 # neither one matches, this time is not the right time. 316 if not (check(self.dayOfMonth, timetuple[2]) or 317 check(self.dayOfWeek, timetuple[6])): 318 return False 319 else: 320 if not check(self.dayOfMonth, timetuple[2]): 321 return False 322 323 if not check(self.dayOfWeek, timetuple[6]): 324 return False 325 326 return True 327 328 while not isRunTime(dateTime): 329 dateTime = addTime(dateTime, 60) 330 assert dateTime[0] < yearLimit, 'Something is wrong with this code' 331 return defer.succeed(time.mktime(dateTime)) 332 333 @defer.deferredGenerator

334 - def startBuild(self):

335 scheds = self.master.db.schedulers 336 # if onlyIfChanged is True, then we will skip this build if no 337 # important changes have occurred since the last invocation 338 if self.onlyIfChanged: 339 wfd = defer.waitForDeferred(scheds.getChangeClassifications(self.objectid)) 340 yield wfd 341 classifications = wfd.getResult() 342 343 # see if we have any important changes 344 for imp in classifications.itervalues(): 345 if imp: 346 break 347 else: 348 log.msg(("Nightly Scheduler <%s>: skipping build " + 349 "- No important changes on configured branch") % self.name) 350 return 351 352 changeids = sorted(classifications.keys()) 353 wfd = defer.waitForDeferred( 354 self.addBuildsetForChanges(reason=self.reason, changeids=changeids)) 355 yield wfd 356 wfd.getResult() 357 358 max_changeid = changeids[-1] # (changeids are sorted) 359 wfd = defer.waitForDeferred( 360 scheds.flushChangeClassifications(self.objectid, 361 less_than=max_changeid+1)) 362 yield wfd 363 wfd.getResult() 364 else: 365 # start a build of the latest revision, whatever that is 366 wfd = defer.waitForDeferred( 367 self.addBuildsetForLatest(reason=self.reason, branch=self.branch)) 368 yield wfd 369 wfd.getResult()

370

Source Code for Module buildbot.schedulers.timed