buildbot.steps.blocker

28 """ 29 Build step that blocks until at least one other step finishes. 30 31 @ivar upstreamSteps: a non-empty list of (builderName, stepName) tuples 32 identifying the other build steps that must 33 complete in order to unblock this Blocker. 34 @ivar idlePolicy: string: what to do if one of the upstream builders is 35 idle when this Blocker starts; one of: 36 \"error\": just blow up (the Blocker will fail with 37 status EXCEPTION) 38 \"ignore\": carry on as if the referenced build step 39 was not mentioned (or is already complete) 40 \"block\": block until the referenced builder starts 41 a build, and then block until the referenced build 42 step in that build finishes 43 @ivar timeout: int: how long to block, in seconds, before giving up and 44 failing (default: None, meaning block forever) 45 """ 46 parms = (BuildStep.parms + 47 ['upstreamSteps', 48 'idlePolicy', 49 'timeout', 50 ]) 51 52 flunkOnFailure = True # override BuildStep's default 53 upstreamSteps = None 54 idlePolicy = "block" 55 timeout = None 56 57 VALID_IDLE_POLICIES = ("error", "ignore", "block") 58

59 - def __init__(self, **kwargs):

60 BuildStep.__init__(self, **kwargs) 61 if self.upstreamSteps is None: 62 config.error( 63 "you must supply upstreamSteps") 64 if len(self.upstreamSteps) < 1: 65 config.error( 66 "upstreamSteps must be a non-empty list") 67 if self.idlePolicy not in self.VALID_IDLE_POLICIES: 68 config.error( 69 "invalid value for idlePolicy: %r (must be one of %s)" 70 % (self.idlePolicy, 71 ", ".join(map(repr, self.VALID_IDLE_POLICIES)))) 72 73 # list of build steps (as BuildStepStatus objects) that we're 74 # currently waiting on 75 self._blocking_steps = [] 76 77 # set of builders (as BuilderStatus objects) that have to start 78 # a Build before we can block on one of their BuildSteps 79 self._blocking_builders = set() 80 81 self._overall_code = builder.SUCCESS # assume the best 82 self._overall_text = [] 83 84 self._timer = None # object returned by reactor.callLater() 85 self._timed_out = False

86

87 - def __str__(self):

88 return self.name

89

90 - def __repr__(self):

91 return "<%s %x: %s>" % (self.__class__.__name__, id(self), self.name)

92

93 - def _log(self, message, *args):

94 log.msg(repr(self) + ": " + (message % args))

95

96 - def buildsMatch(self, buildStatus1, buildStatus2):

97 """ 98 Return true if buildStatus1 and buildStatus2 are from related 99 builds, i.e. a Blocker step running in buildStatus2 should be 100 blocked by an upstream step in buildStatus1. Return false if 101 they are unrelated. 102 103 Default implementation simply raises NotImplementedError: you 104 *must* subclass Blocker and implement this method, because 105 BuildBot currently provides no way to relate different builders. 106 This might change if ticket #875 (\"build flocks\") is 107 implemented. 108 """ 109 raise NotImplementedError( 110 "abstract method: you must subclass Blocker " 111 "and implement buildsMatch()")

112

113 - def _getBuildStatus(self, botmaster, builderName):

114 try: 115 # Get the buildbot.process.builder.Builder object for the 116 # requested upstream builder: this is a long-lived object 117 # that exists and has useful info in it whether or not a 118 # build is currently running under it. 119 builder = botmaster.builders[builderName] 120 except KeyError: 121 raise BadStepError( 122 "no builder named %r" % builderName) 123 124 # The Builder's BuilderStatus object is where we can find out 125 # what's going on right now ... like, say, the list of 126 # BuildStatus objects representing any builds running now. 127 myBuildStatus = self.build.getStatus() 128 builderStatus = builder.builder_status 129 matchingBuild = None 130 131 # Get a list of all builds in this builder, past and present. 132 # This is subtly broken because BuilderStatus does not expose 133 # the information we need; in fact, it doesn't even necessarily 134 # *have* that information. The contents of the build cache can 135 # change unpredictably if someone happens to view the waterfall 136 # at an inopportune moment: yikes! The right fix is to keep old 137 # builds in the database and for BuilderStatus to expose the 138 # needed information. When that is implemented, then Blocker 139 # needs to be adapted to use it, and *then* Blocker should be 140 # safe to use. 141 all_builds = (builderStatus.buildCache.values() + 142 builderStatus.getCurrentBuilds()) 143 144 for buildStatus in all_builds: 145 if self.buildsMatch(myBuildStatus, buildStatus): 146 matchingBuild = buildStatus 147 break 148 149 if matchingBuild is None: 150 msg = "no matching builds found in builder %r" % builderName 151 if self.idlePolicy == "error": 152 raise BadStepError(msg + " (is it idle?)") 153 elif self.idlePolicy == "ignore": 154 # don't hang around waiting (assume the build has finished) 155 self._log(msg + ": skipping it") 156 return None 157 elif self.idlePolicy == "block": 158 self._log(msg + ": will block until it starts a build") 159 self._blocking_builders.add(builderStatus) 160 return None 161 162 self._log("found builder %r: %r", builderName, builder) 163 return matchingBuild

164

165 - def _getStepStatus(self, buildStatus, stepName):

166 for step_status in buildStatus.getSteps(): 167 if step_status.name == stepName: 168 self._log("found build step %r in builder %r: %r", 169 stepName, buildStatus.getBuilder().getName(), step_status) 170 return step_status 171 raise BadStepError( 172 "builder %r has no step named %r" 173 % (buildStatus.builder.name, stepName))

174

175 - def _getFullnames(self):

176 if len(self.upstreamSteps) == 1: 177 fullnames = ["(%s:%s)" % self.upstreamSteps[0]] 178 else: 179 fullnames = [] 180 fullnames.append("(%s:%s," % self.upstreamSteps[0]) 181 fullnames.extend(["%s:%s," % pair for pair in self.upstreamSteps[1:-1]]) 182 fullnames.append("%s:%s)" % self.upstreamSteps[-1]) 183 return fullnames

184

185 - def _getBlockingStatusText(self):

186 return [self.name+":", "blocking on"] + self._getFullnames()

187

188 - def _getFinishStatusText(self, code, elapsed):

189 meaning = builder.Results[code] 190 text = [self.name+":", 191 "upstream %s" % meaning, 192 "after %.1f sec" % elapsed] 193 if code != builder.SUCCESS: 194 text += self._getFullnames() 195 return text

196

197 - def _getTimeoutStatusText(self):

198 return [self.name+":", "timed out", "(%.1f sec)" % self.timeout]

199

200 - def start(self):

201 self.step_status.setText(self._getBlockingStatusText()) 202 203 if self.timeout is not None: 204 self._timer = reactor.callLater(self.timeout, self._timeoutExpired) 205 206 self._log("searching for upstream build steps") 207 botmaster = self.build.slavebuilder.slave.parent 208 errors = [] # list of strings 209 for (builderName, stepName) in self.upstreamSteps: 210 buildStatus = stepStatus = None 211 try: 212 buildStatus = self._getBuildStatus(botmaster, builderName) 213 if buildStatus is not None: 214 stepStatus = self._getStepStatus(buildStatus, stepName) 215 except BadStepError, err: 216 errors.append(err.message) 217 if stepStatus is not None: 218 # Make sure newly-discovered blocking steps are all 219 # added to _blocking_steps before we subscribe to their 220 # "finish" events! 221 self._blocking_steps.append(stepStatus) 222 223 if len(errors) == 1: 224 raise BadStepError(errors[0]) 225 elif len(errors) > 1: 226 raise BadStepError("multiple errors:\n" + "\n".join(errors)) 227 228 self._log("will block on %d upstream build steps: %r", 229 len(self._blocking_steps), self._blocking_steps) 230 if self._blocking_builders: 231 self._log("will also block on %d builders starting a build: %r", 232 len(self._blocking_builders), self._blocking_builders) 233 234 # Now we can register with each blocking step (BuildStepStatus 235 # objects, actually) that we want a callback when the step 236 # finishes. Need to iterate over a copy of _blocking_steps 237 # because it can be modified while we iterate: if any upstream 238 # step is already finished, the _upstreamStepFinished() callback 239 # will be called immediately. 240 for stepStatus in self._blocking_steps[:]: 241 self._awaitStepFinished(stepStatus) 242 self._log("after registering for each upstream build step, " 243 "_blocking_steps = %r", 244 self._blocking_steps) 245 246 # Subscribe to each builder that we're waiting on to start. 247 for bs in self._blocking_builders: 248 bs.subscribe(BuilderStatusReceiver(self, bs))

249

250 - def _awaitStepFinished(self, stepStatus):

251 # N.B. this will callback *immediately* (i.e. even before we 252 # relinquish control to the reactor) if the upstream step in 253 # question has already finished. 254 d = stepStatus.waitUntilFinished() 255 d.addCallback(self._upstreamStepFinished)

256

257 - def _timeoutExpired(self):

258 # Hmmm: this step has failed. But it is still subscribed to 259 # various upstream events, so if they happen despite this 260 # timeout, various callbacks in this object will still be 261 # called. This could be confusing and is definitely a bit 262 # untidy: probably we should unsubscribe from all those various 263 # events. Even better if we had a test case to ensure that we 264 # really do. 265 self._log("timeout (%.1f sec) expired", self.timeout) 266 self.step_status.setColor("red") 267 self.step_status.setText(self._getTimeoutStatusText()) 268 self.finished(builder.FAILURE) 269 self._timed_out = True

270

271 - def _upstreamStepFinished(self, stepStatus):

272 assert isinstance(stepStatus, buildstep.BuildStepStatus) 273 self._log("upstream build step %s:%s finished; results=%r", 274 stepStatus.getBuild().builder.getName(), 275 stepStatus.getName(), 276 stepStatus.getResults()) 277 278 if self._timed_out: 279 # don't care about upstream steps: just clean up and get out 280 self._blocking_steps.remove(stepStatus) 281 return 282 283 (code, text) = stepStatus.getResults() 284 if code != builder.SUCCESS and self._overall_code == builder.SUCCESS: 285 # first non-SUCCESS result wins 286 self._overall_code = code 287 self._overall_text.extend(text) 288 self._log("now _overall_code=%r, _overall_text=%r", 289 self._overall_code, self._overall_text) 290 291 self._blocking_steps.remove(stepStatus) 292 self._checkFinished()

293

294 - def _upstreamBuildStarted(self, builderStatus, buildStatus, receiver):

295 assert isinstance(builderStatus, builder.BuilderStatus) 296 self._log("builder %r (%r) started a build; buildStatus=%r", 297 builderStatus, builderStatus.getName(), buildStatus) 298 299 myBuildStatus = self.build.getStatus() 300 if not self.buildsMatch(myBuildStatus, buildStatus): 301 self._log("but the just-started build does not match: " 302 "ignoring it") 303 return 304 305 builderStatus.unsubscribe(receiver) 306 307 # Need to accumulate newly-discovered steps separately, so we 308 # can add them to _blocking_steps en masse before subscribing to 309 # their "finish" events. 310 new_blocking_steps = [] 311 for (builderName, stepName) in self.upstreamSteps: 312 if builderName == builderStatus.getName(): 313 try: 314 stepStatus = self._getStepStatus(buildStatus, stepName) 315 except BadStepError: 316 self.failed(failure.Failure()) 317 #log.err() 318 #self._overall_code = builder.EXCEPTION 319 #self._overall_text.append(str(err)) 320 else: 321 new_blocking_steps.append(stepStatus) 322 323 self._blocking_steps.extend(new_blocking_steps) 324 for stepStatus in new_blocking_steps: 325 self._awaitStepFinished(stepStatus) 326 327 self._blocking_builders.remove(builderStatus) 328 self._checkFinished()

329

330 - def _checkFinished(self):

331 if self.step_status.isFinished(): 332 # this can happen if _upstreamBuildStarted() catches BadStepError 333 # and fails the step 334 self._log("_checkFinished: already finished, so nothing to do here") 335 return 336 337 self._log("_checkFinished: _blocking_steps=%r, _blocking_builders=%r", 338 self._blocking_steps, self._blocking_builders) 339 340 if not self._blocking_steps and not self._blocking_builders: 341 if self.timeout: 342 self._timer.cancel() 343 344 self.finished(self._overall_code) 345 self.step_status.setText2(self._overall_text) 346 (start, finish) = self.step_status.getTimes() 347 self.step_status.setText( 348 self._getFinishStatusText(self._overall_code, finish - start))

Source Code for Module buildbot.steps.blocker