Package buildbot :: Package steps :: Module blocker
[frames] | no frames]

Source Code for Module buildbot.steps.blocker

  1  # This file is part of Buildbot.  Buildbot is free software: you can 
  2  # redistribute it and/or modify it under the terms of the GNU General Public 
  3  # License as published by the Free Software Foundation, version 2. 
  4  # 
  5  # This program is distributed in the hope that it will be useful, but WITHOUT 
  6  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
  7  # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more 
  8  # details. 
  9  # 
 10  # You should have received a copy of the GNU General Public License along with 
 11  # this program; if not, write to the Free Software Foundation, Inc., 51 
 12  # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 
 13  # 
 14  # Copyright Buildbot Team Members 
 15   
 16  from twisted.python import log, failure 
 17  from twisted.internet import reactor 
 18  from buildbot.process.buildstep import BuildStep 
 19  from buildbot.status import builder, buildstep 
 20  from buildbot import config 
 21   
22 -class BadStepError(Exception):
23 """Raised by Blocker when it is passed an upstream step that cannot 24 be found or is in a bad state.""" 25 pass
26
27 -class Blocker(BuildStep):
28 """ 29 Build step that blocks until at least one other step finishes. 30 31 @ivar upstreamSteps: a non-empty list of (builderName, stepName) tuples 32 identifying the other build steps that must 33 complete in order to unblock this Blocker. 34 @ivar idlePolicy: string: what to do if one of the upstream builders is 35 idle when this Blocker starts; one of: 36 \"error\": just blow up (the Blocker will fail with 37 status EXCEPTION) 38 \"ignore\": carry on as if the referenced build step 39 was not mentioned (or is already complete) 40 \"block\": block until the referenced builder starts 41 a build, and then block until the referenced build 42 step in that build finishes 43 @ivar timeout: int: how long to block, in seconds, before giving up and 44 failing (default: None, meaning block forever) 45 """ 46 parms = (BuildStep.parms + 47 ['upstreamSteps', 48 'idlePolicy', 49 'timeout', 50 ]) 51 52 flunkOnFailure = True # override BuildStep's default 53 upstreamSteps = None 54 idlePolicy = "block" 55 timeout = None 56 57 VALID_IDLE_POLICIES = ("error", "ignore", "block") 58
59 - def __init__(self, **kwargs):
60 BuildStep.__init__(self, **kwargs) 61 if self.upstreamSteps is None: 62 config.error( 63 "you must supply upstreamSteps") 64 if len(self.upstreamSteps) < 1: 65 config.error( 66 "upstreamSteps must be a non-empty list") 67 if self.idlePolicy not in self.VALID_IDLE_POLICIES: 68 config.error( 69 "invalid value for idlePolicy: %r (must be one of %s)" 70 % (self.idlePolicy, 71 ", ".join(map(repr, self.VALID_IDLE_POLICIES)))) 72 73 # list of build steps (as BuildStepStatus objects) that we're 74 # currently waiting on 75 self._blocking_steps = [] 76 77 # set of builders (as BuilderStatus objects) that have to start 78 # a Build before we can block on one of their BuildSteps 79 self._blocking_builders = set() 80 81 self._overall_code = builder.SUCCESS # assume the best 82 self._overall_text = [] 83 84 self._timer = None # object returned by reactor.callLater() 85 self._timed_out = False
86
87 - def __str__(self):
88 return self.name
89
90 - def __repr__(self):
91 return "<%s %x: %s>" % (self.__class__.__name__, id(self), self.name)
92
93 - def _log(self, message, *args):
94 log.msg(repr(self) + ": " + (message % args))
95
96 - def buildsMatch(self, buildStatus1, buildStatus2):
97 """ 98 Return true if buildStatus1 and buildStatus2 are from related 99 builds, i.e. a Blocker step running in buildStatus2 should be 100 blocked by an upstream step in buildStatus1. Return false if 101 they are unrelated. 102 103 Default implementation simply raises NotImplementedError: you 104 *must* subclass Blocker and implement this method, because 105 BuildBot currently provides no way to relate different builders. 106 This might change if ticket #875 (\"build flocks\") is 107 implemented. 108 """ 109 raise NotImplementedError( 110 "abstract method: you must subclass Blocker " 111 "and implement buildsMatch()")
112
113 - def _getBuildStatus(self, botmaster, builderName):
114 try: 115 # Get the buildbot.process.builder.Builder object for the 116 # requested upstream builder: this is a long-lived object 117 # that exists and has useful info in it whether or not a 118 # build is currently running under it. 119 builder = botmaster.builders[builderName] 120 except KeyError: 121 raise BadStepError( 122 "no builder named %r" % builderName) 123 124 # The Builder's BuilderStatus object is where we can find out 125 # what's going on right now ... like, say, the list of 126 # BuildStatus objects representing any builds running now. 127 myBuildStatus = self.build.getStatus() 128 builderStatus = builder.builder_status 129 matchingBuild = None 130 131 # Get a list of all builds in this builder, past and present. 132 # This is subtly broken because BuilderStatus does not expose 133 # the information we need; in fact, it doesn't even necessarily 134 # *have* that information. The contents of the build cache can 135 # change unpredictably if someone happens to view the waterfall 136 # at an inopportune moment: yikes! The right fix is to keep old 137 # builds in the database and for BuilderStatus to expose the 138 # needed information. When that is implemented, then Blocker 139 # needs to be adapted to use it, and *then* Blocker should be 140 # safe to use. 141 all_builds = (builderStatus.buildCache.values() + 142 builderStatus.getCurrentBuilds()) 143 144 for buildStatus in all_builds: 145 if self.buildsMatch(myBuildStatus, buildStatus): 146 matchingBuild = buildStatus 147 break 148 149 if matchingBuild is None: 150 msg = "no matching builds found in builder %r" % builderName 151 if self.idlePolicy == "error": 152 raise BadStepError(msg + " (is it idle?)") 153 elif self.idlePolicy == "ignore": 154 # don't hang around waiting (assume the build has finished) 155 self._log(msg + ": skipping it") 156 return None 157 elif self.idlePolicy == "block": 158 self._log(msg + ": will block until it starts a build") 159 self._blocking_builders.add(builderStatus) 160 return None 161 162 self._log("found builder %r: %r", builderName, builder) 163 return matchingBuild
164
165 - def _getStepStatus(self, buildStatus, stepName):
166 for step_status in buildStatus.getSteps(): 167 if step_status.name == stepName: 168 self._log("found build step %r in builder %r: %r", 169 stepName, buildStatus.getBuilder().getName(), step_status) 170 return step_status 171 raise BadStepError( 172 "builder %r has no step named %r" 173 % (buildStatus.builder.name, stepName))
174
175 - def _getFullnames(self):
176 if len(self.upstreamSteps) == 1: 177 fullnames = ["(%s:%s)" % self.upstreamSteps[0]] 178 else: 179 fullnames = [] 180 fullnames.append("(%s:%s," % self.upstreamSteps[0]) 181 fullnames.extend(["%s:%s," % pair for pair in self.upstreamSteps[1:-1]]) 182 fullnames.append("%s:%s)" % self.upstreamSteps[-1]) 183 return fullnames
184
185 - def _getBlockingStatusText(self):
186 return [self.name+":", "blocking on"] + self._getFullnames()
187
188 - def _getFinishStatusText(self, code, elapsed):
189 meaning = builder.Results[code] 190 text = [self.name+":", 191 "upstream %s" % meaning, 192 "after %.1f sec" % elapsed] 193 if code != builder.SUCCESS: 194 text += self._getFullnames() 195 return text
196
197 - def _getTimeoutStatusText(self):
198 return [self.name+":", "timed out", "(%.1f sec)" % self.timeout]
199
200 - def start(self):
201 self.step_status.setText(self._getBlockingStatusText()) 202 203 if self.timeout is not None: 204 self._timer = reactor.callLater(self.timeout, self._timeoutExpired) 205 206 self._log("searching for upstream build steps") 207 botmaster = self.build.slavebuilder.slave.parent 208 errors = [] # list of strings 209 for (builderName, stepName) in self.upstreamSteps: 210 buildStatus = stepStatus = None 211 try: 212 buildStatus = self._getBuildStatus(botmaster, builderName) 213 if buildStatus is not None: 214 stepStatus = self._getStepStatus(buildStatus, stepName) 215 except BadStepError, err: 216 errors.append(err.message) 217 if stepStatus is not None: 218 # Make sure newly-discovered blocking steps are all 219 # added to _blocking_steps before we subscribe to their 220 # "finish" events! 221 self._blocking_steps.append(stepStatus) 222 223 if len(errors) == 1: 224 raise BadStepError(errors[0]) 225 elif len(errors) > 1: 226 raise BadStepError("multiple errors:\n" + "\n".join(errors)) 227 228 self._log("will block on %d upstream build steps: %r", 229 len(self._blocking_steps), self._blocking_steps) 230 if self._blocking_builders: 231 self._log("will also block on %d builders starting a build: %r", 232 len(self._blocking_builders), self._blocking_builders) 233 234 # Now we can register with each blocking step (BuildStepStatus 235 # objects, actually) that we want a callback when the step 236 # finishes. Need to iterate over a copy of _blocking_steps 237 # because it can be modified while we iterate: if any upstream 238 # step is already finished, the _upstreamStepFinished() callback 239 # will be called immediately. 240 for stepStatus in self._blocking_steps[:]: 241 self._awaitStepFinished(stepStatus) 242 self._log("after registering for each upstream build step, " 243 "_blocking_steps = %r", 244 self._blocking_steps) 245 246 # Subscribe to each builder that we're waiting on to start. 247 for bs in self._blocking_builders: 248 bs.subscribe(BuilderStatusReceiver(self, bs))
249
250 - def _awaitStepFinished(self, stepStatus):
251 # N.B. this will callback *immediately* (i.e. even before we 252 # relinquish control to the reactor) if the upstream step in 253 # question has already finished. 254 d = stepStatus.waitUntilFinished() 255 d.addCallback(self._upstreamStepFinished)
256
257 - def _timeoutExpired(self):
258 # Hmmm: this step has failed. But it is still subscribed to 259 # various upstream events, so if they happen despite this 260 # timeout, various callbacks in this object will still be 261 # called. This could be confusing and is definitely a bit 262 # untidy: probably we should unsubscribe from all those various 263 # events. Even better if we had a test case to ensure that we 264 # really do. 265 self._log("timeout (%.1f sec) expired", self.timeout) 266 self.step_status.setColor("red") 267 self.step_status.setText(self._getTimeoutStatusText()) 268 self.finished(builder.FAILURE) 269 self._timed_out = True
270
271 - def _upstreamStepFinished(self, stepStatus):
272 assert isinstance(stepStatus, buildstep.BuildStepStatus) 273 self._log("upstream build step %s:%s finished; results=%r", 274 stepStatus.getBuild().builder.getName(), 275 stepStatus.getName(), 276 stepStatus.getResults()) 277 278 if self._timed_out: 279 # don't care about upstream steps: just clean up and get out 280 self._blocking_steps.remove(stepStatus) 281 return 282 283 (code, text) = stepStatus.getResults() 284 if code != builder.SUCCESS and self._overall_code == builder.SUCCESS: 285 # first non-SUCCESS result wins 286 self._overall_code = code 287 self._overall_text.extend(text) 288 self._log("now _overall_code=%r, _overall_text=%r", 289 self._overall_code, self._overall_text) 290 291 self._blocking_steps.remove(stepStatus) 292 self._checkFinished()
293
294 - def _upstreamBuildStarted(self, builderStatus, buildStatus, receiver):
295 assert isinstance(builderStatus, builder.BuilderStatus) 296 self._log("builder %r (%r) started a build; buildStatus=%r", 297 builderStatus, builderStatus.getName(), buildStatus) 298 299 myBuildStatus = self.build.getStatus() 300 if not self.buildsMatch(myBuildStatus, buildStatus): 301 self._log("but the just-started build does not match: " 302 "ignoring it") 303 return 304 305 builderStatus.unsubscribe(receiver) 306 307 # Need to accumulate newly-discovered steps separately, so we 308 # can add them to _blocking_steps en masse before subscribing to 309 # their "finish" events. 310 new_blocking_steps = [] 311 for (builderName, stepName) in self.upstreamSteps: 312 if builderName == builderStatus.getName(): 313 try: 314 stepStatus = self._getStepStatus(buildStatus, stepName) 315 except BadStepError: 316 self.failed(failure.Failure()) 317 #log.err() 318 #self._overall_code = builder.EXCEPTION 319 #self._overall_text.append(str(err)) 320 else: 321 new_blocking_steps.append(stepStatus) 322 323 self._blocking_steps.extend(new_blocking_steps) 324 for stepStatus in new_blocking_steps: 325 self._awaitStepFinished(stepStatus) 326 327 self._blocking_builders.remove(builderStatus) 328 self._checkFinished()
329
330 - def _checkFinished(self):
331 if self.step_status.isFinished(): 332 # this can happen if _upstreamBuildStarted() catches BadStepError 333 # and fails the step 334 self._log("_checkFinished: already finished, so nothing to do here") 335 return 336 337 self._log("_checkFinished: _blocking_steps=%r, _blocking_builders=%r", 338 self._blocking_steps, self._blocking_builders) 339 340 if not self._blocking_steps and not self._blocking_builders: 341 if self.timeout: 342 self._timer.cancel() 343 344 self.finished(self._overall_code) 345 self.step_status.setText2(self._overall_text) 346 (start, finish) = self.step_status.getTimes() 347 self.step_status.setText( 348 self._getFinishStatusText(self._overall_code, finish - start))
349
350 -class BuilderStatusReceiver:
351 - def __init__(self, blocker, builderStatus):
352 # the Blocker step that wants to find out when a Builder starts 353 # a Build 354 self.blocker = blocker 355 self.builderStatus = builderStatus
356
357 - def builderChangedState(self, *args):
358 pass
359
360 - def buildStarted(self, name, buildStatus):
361 log.msg("BuilderStatusReceiver: " 362 "apparently, builder %r has started build %r" 363 % (name, buildStatus)) 364 self.blocker._upstreamBuildStarted(self.builderStatus, buildStatus, self)
365
366 - def buildFinished(self, *args):
367 pass
368