buildbot.process.base

1 # -*- test-case-name: buildbot.test.test_step -*- 2 3 import types 4 5 from zope.interface import implements 6 from twisted.python import log 7 from twisted.python.failure import Failure 8 from twisted.internet import reactor, defer, error 9 10 from buildbot import interfaces, locks 11 from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, EXCEPTION, \ 12 RETRY, worst_status 13 from buildbot.status.builder import Results 14 from buildbot.status.progress import BuildProgress 15 16

17 -class Build:

18 """I represent a single build by a single slave. Specialized Builders can 19 use subclasses of Build to hold status information unique to those build 20 processes. 21 22 I control B{how} the build proceeds. The actual build is broken up into a 23 series of steps, saved in the .buildSteps[] array as a list of 24 L{buildbot.process.step.BuildStep} objects. Each step is a single remote 25 command, possibly a shell command. 26 27 During the build, I put status information into my C{BuildStatus} 28 gatherer. 29 30 After the build, I go away. 31 32 I can be used by a factory by setting buildClass on 33 L{buildbot.process.factory.BuildFactory} 34 35 @ivar requests: the list of L{BuildRequest}s that triggered me 36 @ivar build_status: the L{buildbot.status.builder.BuildStatus} that 37 collects our status 38 """ 39 40 implements(interfaces.IBuildControl) 41 42 workdir = "build" 43 build_status = None 44 reason = "changes" 45 finished = False 46 results = None 47 stopped = False 48

49 - def __init__(self, requests):

50 self.requests = requests 51 for req in self.requests: 52 req.startCount += 1 53 self.locks = [] 54 # build a source stamp 55 self.source = requests[0].mergeWith(requests[1:]) 56 self.reason = requests[0].mergeReasons(requests[1:]) 57 58 self.progress = None 59 self.currentStep = None 60 self.slaveEnvironment = {} 61 62 self.terminate = False 63 64 self._acquiringLock = None

65

66 - def setBuilder(self, builder):

67 """ 68 Set the given builder as our builder. 69 70 @type builder: L{buildbot.process.builder.Builder} 71 """ 72 self.builder = builder

73

74 - def setLocks(self, locks):

75 self.locks = locks

76

77 - def setSlaveEnvironment(self, env):

78 self.slaveEnvironment = env

79

80 - def getSourceStamp(self):

81 return self.source

82

83 - def setProperty(self, propname, value, source, runtime=True):

84 """Set a property on this build. This may only be called after the 85 build has started, so that it has a BuildStatus object where the 86 properties can live.""" 87 self.build_status.setProperty(propname, value, source, runtime=True)

88

89 - def getProperties(self):

90 return self.build_status.getProperties()

91

92 - def getProperty(self, propname):

93 return self.build_status.getProperty(propname)

94

95 - def allChanges(self):

96 return self.source.changes

97

98 - def allFiles(self):

99 # return a list of all source files that were changed 100 files = [] 101 for c in self.allChanges(): 102 for f in c.files: 103 files.append(f) 104 return files

105

106 - def __repr__(self):

107 return "<Build %s>" % (self.builder.name,)

108

109 - def blamelist(self):

110 blamelist = [] 111 for c in self.allChanges(): 112 if c.who not in blamelist: 113 blamelist.append(c.who) 114 blamelist.sort() 115 return blamelist

116

117 - def changesText(self):

118 changetext = "" 119 for c in self.allChanges(): 120 changetext += "-" * 60 + "\n\n" + c.asText() + "\n" 121 # consider sorting these by number 122 return changetext

123

124 - def setStepFactories(self, step_factories):

125 """Set a list of 'step factories', which are tuples of (class, 126 kwargs), where 'class' is generally a subclass of step.BuildStep . 127 These are used to create the Steps themselves when the Build starts 128 (as opposed to when it is first created). By creating the steps 129 later, their __init__ method will have access to things like 130 build.allFiles() .""" 131 self.stepFactories = list(step_factories)

132 133 134 135 useProgress = True 136

137 - def getSlaveCommandVersion(self, command, oldversion=None):

138 return self.slavebuilder.getSlaveCommandVersion(command, oldversion)

139 - def getSlaveName(self):

140 return self.slavebuilder.slave.slavename

141

142 - def setupProperties(self):

143 props = self.getProperties() 144 145 # start with global properties from the configuration 146 buildmaster = self.builder.botmaster.parent 147 props.updateFromProperties(buildmaster.properties) 148 149 # from the SourceStamp, which has properties via Change 150 for change in self.source.changes: 151 props.updateFromProperties(change.properties) 152 153 # and finally, get any properties from requests (this is the path 154 # through which schedulers will send us properties) 155 for rq in self.requests: 156 props.updateFromProperties(rq.properties) 157 158 # now set some properties of our own, corresponding to the 159 # build itself 160 props.setProperty("buildnumber", self.build_status.number, "Build") 161 props.setProperty("branch", self.source.branch, "Build") 162 props.setProperty("revision", self.source.revision, "Build") 163 props.setProperty("repository", self.source.repository, "Build") 164 props.setProperty("project", self.source.project, "Build") 165 self.builder.setupProperties(props)

166

167 - def setupSlaveBuilder(self, slavebuilder):

168 self.slavebuilder = slavebuilder 169 170 # navigate our way back to the L{buildbot.buildslave.BuildSlave} 171 # object that came from the config, and get its properties 172 buildslave_properties = slavebuilder.slave.properties 173 self.getProperties().updateFromProperties(buildslave_properties) 174 175 self.slavename = slavebuilder.slave.slavename 176 self.build_status.setSlavename(self.slavename)

177

178 - def startBuild(self, build_status, expectations, slavebuilder):

179 """This method sets up the build, then starts it by invoking the 180 first Step. It returns a Deferred which will fire when the build 181 finishes. This Deferred is guaranteed to never errback.""" 182 183 # we are taking responsibility for watching the connection to the 184 # remote. This responsibility was held by the Builder until our 185 # startBuild was called, and will not return to them until we fire 186 # the Deferred returned by this method. 187 188 log.msg("%s.startBuild" % self) 189 self.build_status = build_status 190 # now that we have a build_status, we can set properties 191 self.setupProperties() 192 self.setupSlaveBuilder(slavebuilder) 193 slavebuilder.slave.updateSlaveStatus(buildStarted=build_status) 194 195 # convert all locks into their real forms 196 lock_list = [] 197 for access in self.locks: 198 if not isinstance(access, locks.LockAccess): 199 # Buildbot 0.7.7 compability: user did not specify access 200 access = access.defaultAccess() 201 lock = self.builder.botmaster.getLockByID(access.lockid) 202 lock_list.append((lock, access)) 203 self.locks = lock_list 204 # then narrow SlaveLocks down to the right slave 205 self.locks = [(l.getLock(self.slavebuilder), la) 206 for l, la in self.locks] 207 self.remote = slavebuilder.remote 208 self.remote.notifyOnDisconnect(self.lostRemote) 209 d = self.deferred = defer.Deferred() 210 def _release_slave(res, slave, bs): 211 self.slavebuilder.buildFinished() 212 slave.updateSlaveStatus(buildFinished=bs) 213 return res

214 d.addCallback(_release_slave, self.slavebuilder.slave, build_status) 215 216 try: 217 self.setupBuild(expectations) # create .steps 218 except: 219 # the build hasn't started yet, so log the exception as a point 220 # event instead of flunking the build. 221 # TODO: associate this failure with the build instead. 222 # this involves doing 223 # self.build_status.buildStarted() from within the exception 224 # handler 225 log.msg("Build.setupBuild failed") 226 log.err(Failure()) 227 self.builder.builder_status.addPointEvent(["setupBuild", 228 "exception"]) 229 self.finished = True 230 self.results = FAILURE 231 self.deferred = None 232 d.callback(self) 233 return d 234 235 self.build_status.buildStarted(self) 236 self.acquireLocks().addCallback(self._startBuild_2) 237 return d

238

239 - def acquireLocks(self, res=None):

240 self._acquiringLock = None 241 if not self.locks: 242 return defer.succeed(None) 243 if self.stopped: 244 return defer.succeed(None) 245 log.msg("acquireLocks(build %s, locks %s)" % (self, self.locks)) 246 for lock, access in self.locks: 247 if not lock.isAvailable(access): 248 log.msg("Build %s waiting for lock %s" % (self, lock)) 249 d = lock.waitUntilMaybeAvailable(self, access) 250 d.addCallback(self.acquireLocks) 251 self._acquiringLock = (lock, access, d) 252 return d 253 # all locks are available, claim them all 254 for lock, access in self.locks: 255 lock.claim(self, access) 256 return defer.succeed(None)

257

258 - def _startBuild_2(self, res):

259 self.startNextStep()

260

261 - def setupBuild(self, expectations):

262 # create the actual BuildSteps. If there are any name collisions, we 263 # add a count to the loser until it is unique. 264 self.steps = [] 265 self.stepStatuses = {} 266 stepnames = {} 267 sps = [] 268 269 for factory, args in self.stepFactories: 270 args = args.copy() 271 try: 272 step = factory(**args) 273 except: 274 log.msg("error while creating step, factory=%s, args=%s" 275 % (factory, args)) 276 raise 277 step.setBuild(self) 278 step.setBuildSlave(self.slavebuilder.slave) 279 if callable (self.workdir): 280 step.setDefaultWorkdir (self.workdir (self.source)) 281 else: 282 step.setDefaultWorkdir (self.workdir) 283 name = step.name 284 if stepnames.has_key(name): 285 count = stepnames[name] 286 count += 1 287 stepnames[name] = count 288 name = step.name + "_%d" % count 289 else: 290 stepnames[name] = 0 291 step.name = name 292 self.steps.append(step) 293 294 # tell the BuildStatus about the step. This will create a 295 # BuildStepStatus and bind it to the Step. 296 step_status = self.build_status.addStepWithName(name) 297 step.setStepStatus(step_status) 298 299 sp = None 300 if self.useProgress: 301 # XXX: maybe bail if step.progressMetrics is empty? or skip 302 # progress for that one step (i.e. "it is fast"), or have a 303 # separate "variable" flag that makes us bail on progress 304 # tracking 305 sp = step.setupProgress() 306 if sp: 307 sps.append(sp) 308 309 # Create a buildbot.status.progress.BuildProgress object. This is 310 # called once at startup to figure out how to build the long-term 311 # Expectations object, and again at the start of each build to get a 312 # fresh BuildProgress object to track progress for that individual 313 # build. TODO: revisit at-startup call 314 315 if self.useProgress: 316 self.progress = BuildProgress(sps) 317 if self.progress and expectations: 318 self.progress.setExpectationsFrom(expectations) 319 320 # we are now ready to set up our BuildStatus. 321 self.build_status.setSourceStamp(self.source) 322 self.build_status.setReason(self.reason) 323 self.build_status.setBlamelist(self.blamelist()) 324 self.build_status.setProgress(self.progress) 325 326 # gather owners from build requests 327 owners = [r.properties['owner'] for r in self.requests 328 if r.properties.has_key('owner')] 329 if owners: self.setProperty('owners', owners, self.reason) 330 331 self.results = [] # list of FAILURE, SUCCESS, WARNINGS, SKIPPED 332 self.result = SUCCESS # overall result, may downgrade after each step 333 self.text = [] # list of text string lists (text2)

334

335 - def getNextStep(self):

336 """This method is called to obtain the next BuildStep for this build. 337 When it returns None (or raises a StopIteration exception), the build 338 is complete.""" 339 if self.stopped: 340 return None 341 if not self.steps: 342 return None 343 if not self.remote: 344 return None 345 if self.terminate: 346 while True: 347 s = self.steps.pop(0) 348 if s.alwaysRun: 349 return s 350 if not self.steps: 351 return None 352 else: 353 return self.steps.pop(0)

354

355 - def startNextStep(self):

356 try: 357 s = self.getNextStep() 358 except StopIteration: 359 s = None 360 if not s: 361 return self.allStepsDone() 362 self.currentStep = s 363 d = defer.maybeDeferred(s.startStep, self.remote) 364 d.addCallback(self._stepDone, s) 365 d.addErrback(self.buildException)

366

367 - def _stepDone(self, results, step):

368 self.currentStep = None 369 if self.finished: 370 return # build was interrupted, don't keep building 371 terminate = self.stepDone(results, step) # interpret/merge results 372 if terminate: 373 self.terminate = True 374 return self.startNextStep()

375

376 - def stepDone(self, result, step):

377 """This method is called when the BuildStep completes. It is passed a 378 status object from the BuildStep and is responsible for merging the 379 Step's results into those of the overall Build.""" 380 381 terminate = False 382 text = None 383 if type(result) == types.TupleType: 384 result, text = result 385 assert type(result) == type(SUCCESS) 386 log.msg(" step '%s' complete: %s" % (step.name, Results[result])) 387 self.results.append(result) 388 if text: 389 self.text.extend(text) 390 if not self.remote: 391 terminate = True 392 393 possible_overall_result = result 394 if result == FAILURE: 395 if not step.flunkOnFailure: 396 possible_overall_result = SUCCESS 397 if step.warnOnFailure: 398 possible_overall_result = WARNINGS 399 if step.flunkOnFailure: 400 possible_overall_result = FAILURE 401 if step.haltOnFailure: 402 terminate = True 403 elif result == WARNINGS: 404 if not step.warnOnWarnings: 405 possible_overall_result = SUCCESS 406 else: 407 possible_overall_result = WARNINGS 408 if step.flunkOnWarnings: 409 possible_overall_result = FAILURE 410 elif result in (EXCEPTION, RETRY): 411 terminate = True 412 413 self.result = worst_status(self.result, possible_overall_result) 414 return terminate

415

416 - def lostRemote(self, remote=None):

417 # the slave went away. There are several possible reasons for this, 418 # and they aren't necessarily fatal. For now, kill the build, but 419 # TODO: see if we can resume the build when it reconnects. 420 log.msg("%s.lostRemote" % self) 421 self.remote = None 422 if self.currentStep: 423 # this should cause the step to finish. 424 log.msg(" stopping currentStep", self.currentStep) 425 self.currentStep.interrupt(Failure(error.ConnectionLost()))

426

427 - def stopBuild(self, reason="<no reason given>"):

428 # the idea here is to let the user cancel a build because, e.g., 429 # they realized they committed a bug and they don't want to waste 430 # the time building something that they know will fail. Another 431 # reason might be to abandon a stuck build. We want to mark the 432 # build as failed quickly rather than waiting for the slave's 433 # timeout to kill it on its own. 434 435 log.msg(" %s: stopping build: %s" % (self, reason)) 436 if self.finished: 437 return 438 # TODO: include 'reason' in this point event 439 self.builder.builder_status.addPointEvent(['interrupt']) 440 self.stopped = True 441 if self.currentStep: 442 self.currentStep.interrupt(reason) 443 444 self.result = FAILURE 445 self.text.append("Interrupted") 446 447 if self._acquiringLock: 448 lock, access, d = self._acquiringLock 449 lock.stopWaitingUntilAvailable(self, access, d) 450 d.callback(None)

451

452 - def allStepsDone(self):

453 if self.result == FAILURE: 454 text = ["failed"] 455 elif self.result == WARNINGS: 456 text = ["warnings"] 457 elif self.result == EXCEPTION: 458 text = ["exception"] 459 else: 460 text = ["build", "successful"] 461 text.extend(self.text) 462 return self.buildFinished(text, self.result)

463

464 - def buildException(self, why):

465 log.msg("%s.buildException" % self) 466 log.err(why) 467 self.buildFinished(["build", "exception"], FAILURE)

468

469 - def buildFinished(self, text, results):

470 """This method must be called when the last Step has completed. It 471 marks the Build as complete and returns the Builder to the 'idle' 472 state. 473 474 It takes two arguments which describe the overall build status: 475 text, results. 'results' is one of SUCCESS, WARNINGS, or FAILURE. 476 477 If 'results' is SUCCESS or WARNINGS, we will permit any dependant 478 builds to start. If it is 'FAILURE', those builds will be 479 abandoned.""" 480 481 self.finished = True 482 if self.remote: 483 self.remote.dontNotifyOnDisconnect(self.lostRemote) 484 self.results = results 485 486 log.msg(" %s: build finished" % self) 487 self.build_status.setText(text) 488 self.build_status.setResults(results) 489 self.build_status.buildFinished() 490 if self.progress and results == SUCCESS: 491 # XXX: also test a 'timing consistent' flag? 492 log.msg(" setting expectations for next time") 493 self.builder.setExpectations(self.progress) 494 reactor.callLater(0, self.releaseLocks) 495 self.deferred.callback(self) 496 self.deferred = None

497

498 - def releaseLocks(self):

499 if self.locks: 500 log.msg("releaseLocks(%s): %s" % (self, self.locks)) 501 for lock, access in self.locks: 502 if lock.isOwner(self, access): 503 lock.release(self, access) 504 else: 505 # This should only happen if we've been interrupted 506 assert self.stopped

507 508 # IBuildControl 509

510 - def getStatus(self):

511 return self.build_status

512 513 # stopBuild is defined earlier 514

Source Code for Module buildbot.process.base