buildbot.process.base

1 # -*- test-case-name: buildbot.test.test_step -*- 2 3 import types 4 5 from zope.interface import implements 6 from twisted.python import log 7 from twisted.python.failure import Failure 8 from twisted.internet import reactor, defer, error 9 10 from buildbot import interfaces, locks 11 from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, EXCEPTION, RETRY 12 from buildbot.status.builder import Results 13 from buildbot.status.progress import BuildProgress 14 15

16 -class Build:

17 """I represent a single build by a single slave. Specialized Builders can 18 use subclasses of Build to hold status information unique to those build 19 processes. 20 21 I control B{how} the build proceeds. The actual build is broken up into a 22 series of steps, saved in the .buildSteps[] array as a list of 23 L{buildbot.process.step.BuildStep} objects. Each step is a single remote 24 command, possibly a shell command. 25 26 During the build, I put status information into my C{BuildStatus} 27 gatherer. 28 29 After the build, I go away. 30 31 I can be used by a factory by setting buildClass on 32 L{buildbot.process.factory.BuildFactory} 33 34 @ivar requests: the list of L{BuildRequest}s that triggered me 35 @ivar build_status: the L{buildbot.status.builder.BuildStatus} that 36 collects our status 37 """ 38 39 implements(interfaces.IBuildControl) 40 41 workdir = "build" 42 build_status = None 43 reason = "changes" 44 finished = False 45 results = None 46 stopped = False 47

48 - def __init__(self, requests):

49 self.requests = requests 50 for req in self.requests: 51 req.startCount += 1 52 self.locks = [] 53 # build a source stamp 54 self.source = requests[0].mergeWith(requests[1:]) 55 self.reason = requests[0].mergeReasons(requests[1:]) 56 57 self.progress = None 58 self.currentStep = None 59 self.slaveEnvironment = {} 60 61 self.terminate = False

62

63 - def setBuilder(self, builder):

64 """ 65 Set the given builder as our builder. 66 67 @type builder: L{buildbot.process.builder.Builder} 68 """ 69 self.builder = builder

70

71 - def setLocks(self, locks):

72 self.locks = locks

73

74 - def setSlaveEnvironment(self, env):

75 self.slaveEnvironment = env

76

77 - def getSourceStamp(self):

78 return self.source

79

80 - def setProperty(self, propname, value, source):

81 """Set a property on this build. This may only be called after the 82 build has started, so that it has a BuildStatus object where the 83 properties can live.""" 84 self.build_status.setProperty(propname, value, source)

85

86 - def getProperties(self):

87 return self.build_status.getProperties()

88

89 - def getProperty(self, propname):

90 return self.build_status.getProperty(propname)

91

92 - def allChanges(self):

93 return self.source.changes

94

95 - def allFiles(self):

96 # return a list of all source files that were changed 97 files = [] 98 for c in self.allChanges(): 99 for f in c.files: 100 files.append(f) 101 return files

102

103 - def __repr__(self):

104 return "<Build %s>" % (self.builder.name,)

105

106 - def blamelist(self):

107 blamelist = [] 108 for c in self.allChanges(): 109 if c.who not in blamelist: 110 blamelist.append(c.who) 111 blamelist.sort() 112 return blamelist

113

114 - def changesText(self):

115 changetext = "" 116 for c in self.allChanges(): 117 changetext += "-" * 60 + "\n\n" + c.asText() + "\n" 118 # consider sorting these by number 119 return changetext

120

121 - def setStepFactories(self, step_factories):

122 """Set a list of 'step factories', which are tuples of (class, 123 kwargs), where 'class' is generally a subclass of step.BuildStep . 124 These are used to create the Steps themselves when the Build starts 125 (as opposed to when it is first created). By creating the steps 126 later, their __init__ method will have access to things like 127 build.allFiles() .""" 128 self.stepFactories = list(step_factories)

129 130 131 132 useProgress = True 133

134 - def getSlaveCommandVersion(self, command, oldversion=None):

135 return self.slavebuilder.getSlaveCommandVersion(command, oldversion)

136 - def getSlaveName(self):

137 return self.slavebuilder.slave.slavename

138

139 - def setupProperties(self):

140 props = self.getProperties() 141 142 # start with global properties from the configuration 143 buildmaster = self.builder.botmaster.parent 144 props.updateFromProperties(buildmaster.properties) 145 146 # get any properties from requests (this is the path through 147 # which schedulers will send us properties) 148 for rq in self.requests: 149 props.updateFromProperties(rq.properties) 150 151 # and finally, from the SourceStamp, which has properties via Change 152 for change in self.source.changes: 153 props.updateFromProperties(change.properties) 154 155 # now set some properties of our own, corresponding to the 156 # build itself 157 props.setProperty("buildnumber", self.build_status.number, "Build") 158 props.setProperty("branch", self.source.branch, "Build") 159 props.setProperty("revision", self.source.revision, "Build") 160 props.setProperty("repository", self.source.repository, "Build") 161 props.setProperty("project", self.source.project, "Build") 162 self.builder.setupProperties(props)

163

164 - def setupSlaveBuilder(self, slavebuilder):

165 self.slavebuilder = slavebuilder 166 167 # navigate our way back to the L{buildbot.buildslave.BuildSlave} 168 # object that came from the config, and get its properties 169 buildslave_properties = slavebuilder.slave.properties 170 self.getProperties().updateFromProperties(buildslave_properties) 171 172 self.slavename = slavebuilder.slave.slavename 173 self.build_status.setSlavename(self.slavename)

174

175 - def startBuild(self, build_status, expectations, slavebuilder):

176 """This method sets up the build, then starts it by invoking the 177 first Step. It returns a Deferred which will fire when the build 178 finishes. This Deferred is guaranteed to never errback.""" 179 180 # we are taking responsibility for watching the connection to the 181 # remote. This responsibility was held by the Builder until our 182 # startBuild was called, and will not return to them until we fire 183 # the Deferred returned by this method. 184 185 log.msg("%s.startBuild" % self) 186 self.build_status = build_status 187 # now that we have a build_status, we can set properties 188 self.setupProperties() 189 self.setupSlaveBuilder(slavebuilder) 190 slavebuilder.slave.updateSlaveStatus(buildStarted=build_status) 191 192 # convert all locks into their real forms 193 lock_list = [] 194 for access in self.locks: 195 if not isinstance(access, locks.LockAccess): 196 # Buildbot 0.7.7 compability: user did not specify access 197 access = access.defaultAccess() 198 lock = self.builder.botmaster.getLockByID(access.lockid) 199 lock_list.append((lock, access)) 200 self.locks = lock_list 201 # then narrow SlaveLocks down to the right slave 202 self.locks = [(l.getLock(self.slavebuilder), la) 203 for l, la in self.locks] 204 self.remote = slavebuilder.remote 205 self.remote.notifyOnDisconnect(self.lostRemote) 206 d = self.deferred = defer.Deferred() 207 def _release_slave(res, slave, bs): 208 self.slavebuilder.buildFinished() 209 slave.updateSlaveStatus(buildFinished=bs) 210 return res

211 d.addCallback(_release_slave, self.slavebuilder.slave, build_status) 212 213 try: 214 self.setupBuild(expectations) # create .steps 215 except: 216 # the build hasn't started yet, so log the exception as a point 217 # event instead of flunking the build. 218 # TODO: associate this failure with the build instead. 219 # this involves doing 220 # self.build_status.buildStarted() from within the exception 221 # handler 222 log.msg("Build.setupBuild failed") 223 log.err(Failure()) 224 self.builder.builder_status.addPointEvent(["setupBuild", 225 "exception"]) 226 self.finished = True 227 self.results = FAILURE 228 self.deferred = None 229 d.callback(self) 230 return d 231 232 self.acquireLocks().addCallback(self._startBuild_2) 233 return d

234

235 - def acquireLocks(self, res=None):

236 log.msg("acquireLocks(step %s, locks %s)" % (self, self.locks)) 237 if not self.locks: 238 return defer.succeed(None) 239 for lock, access in self.locks: 240 if not lock.isAvailable(access): 241 log.msg("Build %s waiting for lock %s" % (self, lock)) 242 d = lock.waitUntilMaybeAvailable(self, access) 243 d.addCallback(self.acquireLocks) 244 return d 245 # all locks are available, claim them all 246 for lock, access in self.locks: 247 lock.claim(self, access) 248 return defer.succeed(None)

249

250 - def _startBuild_2(self, res):

251 self.build_status.buildStarted(self) 252 self.startNextStep()

253

254 - def setupBuild(self, expectations):

255 # create the actual BuildSteps. If there are any name collisions, we 256 # add a count to the loser until it is unique. 257 self.steps = [] 258 self.stepStatuses = {} 259 stepnames = {} 260 sps = [] 261 262 for factory, args in self.stepFactories: 263 args = args.copy() 264 try: 265 step = factory(**args) 266 except: 267 log.msg("error while creating step, factory=%s, args=%s" 268 % (factory, args)) 269 raise 270 step.setBuild(self) 271 step.setBuildSlave(self.slavebuilder.slave) 272 if callable (self.workdir): 273 step.setDefaultWorkdir (self.workdir (self.source)) 274 else: 275 step.setDefaultWorkdir (self.workdir) 276 name = step.name 277 if stepnames.has_key(name): 278 count = stepnames[name] 279 count += 1 280 stepnames[name] = count 281 name = step.name + "_%d" % count 282 else: 283 stepnames[name] = 0 284 step.name = name 285 self.steps.append(step) 286 287 # tell the BuildStatus about the step. This will create a 288 # BuildStepStatus and bind it to the Step. 289 step_status = self.build_status.addStepWithName(name) 290 step.setStepStatus(step_status) 291 292 sp = None 293 if self.useProgress: 294 # XXX: maybe bail if step.progressMetrics is empty? or skip 295 # progress for that one step (i.e. "it is fast"), or have a 296 # separate "variable" flag that makes us bail on progress 297 # tracking 298 sp = step.setupProgress() 299 if sp: 300 sps.append(sp) 301 302 # Create a buildbot.status.progress.BuildProgress object. This is 303 # called once at startup to figure out how to build the long-term 304 # Expectations object, and again at the start of each build to get a 305 # fresh BuildProgress object to track progress for that individual 306 # build. TODO: revisit at-startup call 307 308 if self.useProgress: 309 self.progress = BuildProgress(sps) 310 if self.progress and expectations: 311 self.progress.setExpectationsFrom(expectations) 312 313 # we are now ready to set up our BuildStatus. 314 self.build_status.setSourceStamp(self.source) 315 self.build_status.setReason(self.reason) 316 self.build_status.setBlamelist(self.blamelist()) 317 self.build_status.setProgress(self.progress) 318 319 # gather owners from build requests 320 owners = [r.properties['owner'] for r in self.requests 321 if r.properties.has_key('owner')] 322 if owners: self.setProperty('owners', owners, self.reason) 323 324 self.results = [] # list of FAILURE, SUCCESS, WARNINGS, SKIPPED 325 self.result = SUCCESS # overall result, may downgrade after each step 326 self.text = [] # list of text string lists (text2)

327

328 - def getNextStep(self):

329 """This method is called to obtain the next BuildStep for this build. 330 When it returns None (or raises a StopIteration exception), the build 331 is complete.""" 332 if self.stopped: 333 return None 334 if not self.steps: 335 return None 336 if self.terminate: 337 while True: 338 s = self.steps.pop(0) 339 if s.alwaysRun: 340 return s 341 if not self.steps: 342 return None 343 else: 344 return self.steps.pop(0)

345

346 - def startNextStep(self):

347 try: 348 s = self.getNextStep() 349 except StopIteration: 350 s = None 351 if not s: 352 return self.allStepsDone() 353 self.currentStep = s 354 d = defer.maybeDeferred(s.startStep, self.remote) 355 d.addCallback(self._stepDone, s) 356 d.addErrback(self.buildException)

357

358 - def _stepDone(self, results, step):

359 self.currentStep = None 360 if self.finished: 361 return # build was interrupted, don't keep building 362 terminate = self.stepDone(results, step) # interpret/merge results 363 if terminate: 364 self.terminate = True 365 return self.startNextStep()

366

367 - def stepDone(self, result, step):

368 """This method is called when the BuildStep completes. It is passed a 369 status object from the BuildStep and is responsible for merging the 370 Step's results into those of the overall Build.""" 371 372 terminate = False 373 text = None 374 if type(result) == types.TupleType: 375 result, text = result 376 assert type(result) == type(SUCCESS) 377 log.msg(" step '%s' complete: %s" % (step.name, Results[result])) 378 self.results.append(result) 379 if text: 380 self.text.extend(text) 381 if not self.remote: 382 terminate = True 383 if result == FAILURE: 384 if step.warnOnFailure: 385 if self.result != FAILURE: 386 self.result = WARNINGS 387 if step.flunkOnFailure: 388 self.result = FAILURE 389 if step.haltOnFailure: 390 terminate = True 391 elif result == WARNINGS: 392 if step.warnOnWarnings: 393 if self.result != FAILURE: 394 self.result = WARNINGS 395 if step.flunkOnWarnings: 396 self.result = FAILURE 397 elif result == EXCEPTION: 398 self.result = EXCEPTION 399 terminate = True 400 elif result == RETRY: 401 self.result = RETRY 402 terminate = True 403 return terminate

404

405 - def lostRemote(self, remote=None):

406 # the slave went away. There are several possible reasons for this, 407 # and they aren't necessarily fatal. For now, kill the build, but 408 # TODO: see if we can resume the build when it reconnects. 409 log.msg("%s.lostRemote" % self) 410 self.remote = None 411 if self.currentStep: 412 # this should cause the step to finish. 413 log.msg(" stopping currentStep", self.currentStep) 414 self.currentStep.interrupt(Failure(error.ConnectionLost()))

415

416 - def stopBuild(self, reason="<no reason given>"):

417 # the idea here is to let the user cancel a build because, e.g., 418 # they realized they committed a bug and they don't want to waste 419 # the time building something that they know will fail. Another 420 # reason might be to abandon a stuck build. We want to mark the 421 # build as failed quickly rather than waiting for the slave's 422 # timeout to kill it on its own. 423 424 log.msg(" %s: stopping build: %s" % (self, reason)) 425 if self.finished: 426 return 427 # TODO: include 'reason' in this point event 428 self.builder.builder_status.addPointEvent(['interrupt']) 429 self.stopped = True 430 self.currentStep.interrupt(reason)

431

432 - def allStepsDone(self):

433 if self.result == FAILURE: 434 text = ["failed"] 435 elif self.result == WARNINGS: 436 text = ["warnings"] 437 elif self.result == EXCEPTION: 438 text = ["exception"] 439 else: 440 text = ["build", "successful"] 441 text.extend(self.text) 442 return self.buildFinished(text, self.result)

443

444 - def buildException(self, why):

445 log.msg("%s.buildException" % self) 446 log.err(why) 447 self.buildFinished(["build", "exception"], FAILURE)

448

449 - def buildFinished(self, text, results):

450 """This method must be called when the last Step has completed. It 451 marks the Build as complete and returns the Builder to the 'idle' 452 state. 453 454 It takes two arguments which describe the overall build status: 455 text, results. 'results' is one of SUCCESS, WARNINGS, or FAILURE. 456 457 If 'results' is SUCCESS or WARNINGS, we will permit any dependant 458 builds to start. If it is 'FAILURE', those builds will be 459 abandoned.""" 460 461 self.finished = True 462 if self.remote: 463 self.remote.dontNotifyOnDisconnect(self.lostRemote) 464 self.results = results 465 466 log.msg(" %s: build finished" % self) 467 self.build_status.setText(text) 468 self.build_status.setResults(results) 469 self.build_status.buildFinished() 470 if self.progress and results == SUCCESS: 471 # XXX: also test a 'timing consistent' flag? 472 log.msg(" setting expectations for next time") 473 self.builder.setExpectations(self.progress) 474 reactor.callLater(0, self.releaseLocks) 475 self.deferred.callback(self) 476 self.deferred = None

477

478 - def releaseLocks(self):

479 log.msg("releaseLocks(%s): %s" % (self, self.locks)) 480 for lock, access in self.locks: 481 lock.release(self, access)

482 483 # IBuildControl 484

485 - def getStatus(self):

486 return self.build_status

487 488 # stopBuild is defined earlier 489

Source Code for Module buildbot.process.base