buildbot.steps.blocker

28 """ 29 Build step that blocks until at least one other step finishes. 30 31 @ivar upstreamSteps: a non-empty list of (builderName, stepName) tuples 32 identifying the other build steps that must 33 complete in order to unblock this Blocker. 34 @ivar idlePolicy: string: what to do if one of the upstream builders is 35 idle when this Blocker starts; one of: 36 \"error\": just blow up (the Blocker will fail with 37 status EXCEPTION) 38 \"ignore\": carry on as if the referenced build step 39 was not mentioned (or is already complete) 40 \"block\": block until the referenced builder starts 41 a build, and then block until the referenced build 42 step in that build finishes 43 @ivar timeout: int: how long to block, in seconds, before giving up and 44 failing (default: None, meaning block forever) 45 """ 46 parms = (BuildStep.parms + 47 ['upstreamSteps', 48 'idlePolicy', 49 'timeout', 50 ]) 51 52 flunkOnFailure = True # override BuildStep's default 53 upstreamSteps = None 54 idlePolicy = "block" 55 timeout = None 56 57 VALID_IDLE_POLICIES = ("error", "ignore", "block") 58

59 - def __init__(self, **kwargs):

60 BuildStep.__init__(self, **kwargs) 61 if self.upstreamSteps is None: 62 raise ValueError("you must supply upstreamSteps") 63 if len(self.upstreamSteps) < 1: 64 raise ValueError("upstreamSteps must be a non-empty list") 65 if self.idlePolicy not in self.VALID_IDLE_POLICIES: 66 raise ValueError( 67 "invalid value for idlePolicy: %r (must be one of %s)" 68 % (self.idlePolicy, 69 ", ".join(map(repr, self.VALID_IDLE_POLICIES)))) 70 71 # list of build steps (as BuildStepStatus objects) that we're 72 # currently waiting on 73 self._blocking_steps = [] 74 75 # set of builders (as BuilderStatus objects) that have to start 76 # a Build before we can block on one of their BuildSteps 77 self._blocking_builders = set() 78 79 self._overall_code = builder.SUCCESS # assume the best 80 self._overall_text = [] 81 82 self._timer = None # object returned by reactor.callLater() 83 self._timed_out = False

84

85 - def __str__(self):

86 return self.name

87

88 - def __repr__(self):

89 return "<%s %x: %s>" % (self.__class__.__name__, id(self), self.name)

90

91 - def _log(self, message, *args):

92 log.msg(repr(self) + ": " + (message % args))

93

94 - def buildsMatch(self, buildStatus1, buildStatus2):

95 """ 96 Return true if buildStatus1 and buildStatus2 are from related 97 builds, i.e. a Blocker step running in buildStatus2 should be 98 blocked by an upstream step in buildStatus1. Return false if 99 they are unrelated. 100 101 Default implementation simply raises NotImplementedError: you 102 *must* subclass Blocker and implement this method, because 103 BuildBot currently provides no way to relate different builders. 104 This might change if ticket #875 (\"build flocks\") is 105 implemented. 106 """ 107 raise NotImplementedError( 108 "abstract method: you must subclass Blocker " 109 "and implement buildsMatch()")

110

111 - def _getBuildStatus(self, botmaster, builderName):

112 try: 113 # Get the buildbot.process.builder.Builder object for the 114 # requested upstream builder: this is a long-lived object 115 # that exists and has useful info in it whether or not a 116 # build is currently running under it. 117 builder = botmaster.builders[builderName] 118 except KeyError: 119 raise BadStepError( 120 "no builder named %r" % builderName) 121 122 # The Builder's BuilderStatus object is where we can find out 123 # what's going on right now ... like, say, the list of 124 # BuildStatus objects representing any builds running now. 125 myBuildStatus = self.build.getStatus() 126 builderStatus = builder.builder_status 127 matchingBuild = None 128 129 # Get a list of all builds in this builder, past and present. 130 # This is subtly broken because BuilderStatus does not expose 131 # the information we need; in fact, it doesn't even necessarily 132 # *have* that information. The contents of the build cache can 133 # change unpredictably if someone happens to view the waterfall 134 # at an inopportune moment: yikes! The right fix is to keep old 135 # builds in the database and for BuilderStatus to expose the 136 # needed information. When that is implemented, then Blocker 137 # needs to be adapted to use it, and *then* Blocker should be 138 # safe to use. 139 all_builds = (builderStatus.buildCache.values() + 140 builderStatus.getCurrentBuilds()) 141 142 for buildStatus in all_builds: 143 if self.buildsMatch(myBuildStatus, buildStatus): 144 matchingBuild = buildStatus 145 break 146 147 if matchingBuild is None: 148 msg = "no matching builds found in builder %r" % builderName 149 if self.idlePolicy == "error": 150 raise BadStepError(msg + " (is it idle?)") 151 elif self.idlePolicy == "ignore": 152 # don't hang around waiting (assume the build has finished) 153 self._log(msg + ": skipping it") 154 return None 155 elif self.idlePolicy == "block": 156 self._log(msg + ": will block until it starts a build") 157 self._blocking_builders.add(builderStatus) 158 return None 159 160 self._log("found builder %r: %r", builderName, builder) 161 return matchingBuild

162

163 - def _getStepStatus(self, buildStatus, stepName):

164 for step_status in buildStatus.getSteps(): 165 if step_status.name == stepName: 166 self._log("found build step %r in builder %r: %r", 167 stepName, buildStatus.getBuilder().getName(), step_status) 168 return step_status 169 raise BadStepError( 170 "builder %r has no step named %r" 171 % (buildStatus.builder.name, stepName))

172

173 - def _getFullnames(self):

174 if len(self.upstreamSteps) == 1: 175 fullnames = ["(%s:%s)" % self.upstreamSteps[0]] 176 else: 177 fullnames = [] 178 fullnames.append("(%s:%s," % self.upstreamSteps[0]) 179 fullnames.extend(["%s:%s," % pair for pair in self.upstreamSteps[1:-1]]) 180 fullnames.append("%s:%s)" % self.upstreamSteps[-1]) 181 return fullnames

182

183 - def _getBlockingStatusText(self):

184 return [self.name+":", "blocking on"] + self._getFullnames()

185

186 - def _getFinishStatusText(self, code, elapsed):

187 meaning = builder.Results[code] 188 text = [self.name+":", 189 "upstream %s" % meaning, 190 "after %.1f sec" % elapsed] 191 if code != builder.SUCCESS: 192 text += self._getFullnames() 193 return text

194

195 - def _getTimeoutStatusText(self):

196 return [self.name+":", "timed out", "(%.1f sec)" % self.timeout]

197

198 - def start(self):

199 self.step_status.setText(self._getBlockingStatusText()) 200 201 if self.timeout is not None: 202 self._timer = reactor.callLater(self.timeout, self._timeoutExpired) 203 204 self._log("searching for upstream build steps") 205 botmaster = self.build.slavebuilder.slave.parent 206 errors = [] # list of strings 207 for (builderName, stepName) in self.upstreamSteps: 208 buildStatus = stepStatus = None 209 try: 210 buildStatus = self._getBuildStatus(botmaster, builderName) 211 if buildStatus is not None: 212 stepStatus = self._getStepStatus(buildStatus, stepName) 213 except BadStepError, err: 214 errors.append(err.message) 215 if stepStatus is not None: 216 # Make sure newly-discovered blocking steps are all 217 # added to _blocking_steps before we subscribe to their 218 # "finish" events! 219 self._blocking_steps.append(stepStatus) 220 221 if len(errors) == 1: 222 raise BadStepError(errors[0]) 223 elif len(errors) > 1: 224 raise BadStepError("multiple errors:\n" + "\n".join(errors)) 225 226 self._log("will block on %d upstream build steps: %r", 227 len(self._blocking_steps), self._blocking_steps) 228 if self._blocking_builders: 229 self._log("will also block on %d builders starting a build: %r", 230 len(self._blocking_builders), self._blocking_builders) 231 232 # Now we can register with each blocking step (BuildStepStatus 233 # objects, actually) that we want a callback when the step 234 # finishes. Need to iterate over a copy of _blocking_steps 235 # because it can be modified while we iterate: if any upstream 236 # step is already finished, the _upstreamStepFinished() callback 237 # will be called immediately. 238 for stepStatus in self._blocking_steps[:]: 239 self._awaitStepFinished(stepStatus) 240 self._log("after registering for each upstream build step, " 241 "_blocking_steps = %r", 242 self._blocking_steps) 243 244 # Subscribe to each builder that we're waiting on to start. 245 for bs in self._blocking_builders: 246 bs.subscribe(BuilderStatusReceiver(self, bs))

247

248 - def _awaitStepFinished(self, stepStatus):

249 # N.B. this will callback *immediately* (i.e. even before we 250 # relinquish control to the reactor) if the upstream step in 251 # question has already finished. 252 d = stepStatus.waitUntilFinished() 253 d.addCallback(self._upstreamStepFinished)

254

255 - def _timeoutExpired(self):

256 # Hmmm: this step has failed. But it is still subscribed to 257 # various upstream events, so if they happen despite this 258 # timeout, various callbacks in this object will still be 259 # called. This could be confusing and is definitely a bit 260 # untidy: probably we should unsubscribe from all those various 261 # events. Even better if we had a test case to ensure that we 262 # really do. 263 self._log("timeout (%.1f sec) expired", self.timeout) 264 self.step_status.setColor("red") 265 self.step_status.setText(self._getTimeoutStatusText()) 266 self.finished(builder.FAILURE) 267 self._timed_out = True

268

269 - def _upstreamStepFinished(self, stepStatus):

270 assert isinstance(stepStatus, buildstep.BuildStepStatus) 271 self._log("upstream build step %s:%s finished; results=%r", 272 stepStatus.getBuild().builder.getName(), 273 stepStatus.getName(), 274 stepStatus.getResults()) 275 276 if self._timed_out: 277 # don't care about upstream steps: just clean up and get out 278 self._blocking_steps.remove(stepStatus) 279 return 280 281 (code, text) = stepStatus.getResults() 282 if code != builder.SUCCESS and self._overall_code == builder.SUCCESS: 283 # first non-SUCCESS result wins 284 self._overall_code = code 285 self._overall_text.extend(text) 286 self._log("now _overall_code=%r, _overall_text=%r", 287 self._overall_code, self._overall_text) 288 289 self._blocking_steps.remove(stepStatus) 290 self._checkFinished()

291

292 - def _upstreamBuildStarted(self, builderStatus, buildStatus, receiver):

293 assert isinstance(builderStatus, builder.BuilderStatus) 294 self._log("builder %r (%r) started a build; buildStatus=%r", 295 builderStatus, builderStatus.getName(), buildStatus) 296 297 myBuildStatus = self.build.getStatus() 298 if not self.buildsMatch(myBuildStatus, buildStatus): 299 self._log("but the just-started build does not match: " 300 "ignoring it") 301 return 302 303 builderStatus.unsubscribe(receiver) 304 305 # Need to accumulate newly-discovered steps separately, so we 306 # can add them to _blocking_steps en masse before subscribing to 307 # their "finish" events. 308 new_blocking_steps = [] 309 for (builderName, stepName) in self.upstreamSteps: 310 if builderName == builderStatus.getName(): 311 try: 312 stepStatus = self._getStepStatus(buildStatus, stepName) 313 except BadStepError: 314 self.failed(failure.Failure()) 315 #log.err() 316 #self._overall_code = builder.EXCEPTION 317 #self._overall_text.append(str(err)) 318 else: 319 new_blocking_steps.append(stepStatus) 320 321 self._blocking_steps.extend(new_blocking_steps) 322 for stepStatus in new_blocking_steps: 323 self._awaitStepFinished(stepStatus) 324 325 self._blocking_builders.remove(builderStatus) 326 self._checkFinished()

327

328 - def _checkFinished(self):

329 if self.step_status.isFinished(): 330 # this can happen if _upstreamBuildStarted() catches BadStepError 331 # and fails the step 332 self._log("_checkFinished: already finished, so nothing to do here") 333 return 334 335 self._log("_checkFinished: _blocking_steps=%r, _blocking_builders=%r", 336 self._blocking_steps, self._blocking_builders) 337 338 if not self._blocking_steps and not self._blocking_builders: 339 if self.timeout: 340 self._timer.cancel() 341 342 self.finished(self._overall_code) 343 self.step_status.setText2(self._overall_text) 344 (start, finish) = self.step_status.getTimes() 345 self.step_status.setText( 346 self._getFinishStatusText(self._overall_code, finish - start))

Source Code for Module buildbot.steps.blocker