buildbot.process.base

1 # -*- test-case-name: buildbot.test.test_step -*- 2 3 import types 4 5 from zope.interface import implements 6 from twisted.python import log 7 from twisted.python.failure import Failure 8 from twisted.internet import reactor, defer, error 9 10 from buildbot import interfaces, locks 11 from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, EXCEPTION 12 from buildbot.status.builder import Results, BuildRequestStatus 13 from buildbot.status.progress import BuildProgress 14 from buildbot.process.properties import Properties 15

16 -class BuildRequest:

17 """I represent a request to a specific Builder to run a single build. 18 19 I have a SourceStamp which specifies what sources I will build. This may 20 specify a specific revision of the source tree (so source.branch, 21 source.revision, and source.patch are used). The .patch attribute is 22 either None or a tuple of (patchlevel, diff), consisting of a number to 23 use in 'patch -pN', and a unified-format context diff. 24 25 Alternatively, the SourceStamp may specify a set of Changes to be built, 26 contained in source.changes. In this case, I may be mergeable with other 27 BuildRequests on the same branch. 28 29 I may be part of a BuildSet, in which case I will report status results 30 to it. 31 32 I am paired with a BuildRequestStatus object, to which I feed status 33 information. 34 35 @type source: a L{buildbot.sourcestamp.SourceStamp} instance. 36 @ivar source: the source code that this BuildRequest use 37 38 @type reason: string 39 @ivar reason: the reason this Build is being requested. Schedulers 40 provide this, but for forced builds the user requesting the 41 build will provide a string. 42 43 @type properties: Properties object 44 @ivar properties: properties that should be applied to this build 45 'owner' property is used by Build objects to collect 46 the list returned by getInterestedUsers 47 48 @ivar status: the IBuildStatus object which tracks our status 49 50 @ivar submittedAt: a timestamp (seconds since epoch) when this request 51 was submitted to the Builder. This is used by the CVS 52 step to compute a checkout timestamp, as well as the 53 master to prioritize build requests from oldest to 54 newest. 55 """ 56 57 source = None 58 builder = None 59 startCount = 0 # how many times we have tried to start this build 60 submittedAt = None 61 62 implements(interfaces.IBuildRequestControl) 63

64 - def __init__(self, reason, source, builderName, properties=None):

65 assert interfaces.ISourceStamp(source, None) 66 self.reason = reason 67 self.source = source 68 69 self.properties = Properties() 70 if properties: 71 self.properties.updateFromProperties(properties) 72 73 self.start_watchers = [] 74 self.finish_watchers = [] 75 self.status = BuildRequestStatus(source, builderName)

76

77 - def canBeMergedWith(self, other):

78 return self.source.canBeMergedWith(other.source)

79

80 - def mergeWith(self, others):

81 return self.source.mergeWith([o.source for o in others])

82

83 - def mergeReasons(self, others):

84 """Return a reason for the merged build request.""" 85 reasons = [] 86 for req in [self] + others: 87 if req.reason and req.reason not in reasons: 88 reasons.append(req.reason) 89 return ", ".join(reasons)

90

91 - def waitUntilFinished(self):

92 """Get a Deferred that will fire (with a 93 L{buildbot.interfaces.IBuildStatus} instance when the build 94 finishes.""" 95 d = defer.Deferred() 96 self.finish_watchers.append(d) 97 return d

98 99 # these are called by the Builder 100

101 - def requestSubmitted(self, builder):

102 # the request has been placed on the queue 103 self.builder = builder

104

105 - def buildStarted(self, build, buildstatus):

106 """This is called by the Builder when a Build has been started in the 107 hopes of satifying this BuildRequest. It may be called multiple 108 times, since interrupted builds and lost buildslaves may force 109 multiple Builds to be run until the fate of the BuildRequest is known 110 for certain.""" 111 for o in self.start_watchers[:]: 112 # these observers get the IBuildControl 113 o(build) 114 # while these get the IBuildStatus 115 self.status.buildStarted(buildstatus)

116

117 - def finished(self, buildstatus):

118 """This is called by the Builder when the BuildRequest has been 119 retired. This happens when its Build has either succeeded (yay!) or 120 failed (boo!). TODO: If it is halted due to an exception (oops!), or 121 some other retryable error, C{finished} will not be called yet.""" 122 123 for w in self.finish_watchers: 124 w.callback(buildstatus) 125 self.finish_watchers = []

126 127 # IBuildRequestControl 128

131 - def unsubscribe(self, observer):

132 self.start_watchers.remove(observer)

133

134 - def cancel(self):

135 """Cancel this request. This can only be successful if the Build has 136 not yet been started. 137 138 @return: a boolean indicating if the cancel was successful.""" 139 if self.builder: 140 return self.builder.cancelBuildRequest(self) 141 return False

142

143 - def setSubmitTime(self, t):

144 self.submittedAt = t 145 self.status.setSubmitTime(t)

146

147 - def getSubmitTime(self):

148 return self.submittedAt

149 150

151 -class Build:

152 """I represent a single build by a single slave. Specialized Builders can 153 use subclasses of Build to hold status information unique to those build 154 processes. 155 156 I control B{how} the build proceeds. The actual build is broken up into a 157 series of steps, saved in the .buildSteps[] array as a list of 158 L{buildbot.process.step.BuildStep} objects. Each step is a single remote 159 command, possibly a shell command. 160 161 During the build, I put status information into my C{BuildStatus} 162 gatherer. 163 164 After the build, I go away. 165 166 I can be used by a factory by setting buildClass on 167 L{buildbot.process.factory.BuildFactory} 168 169 @ivar requests: the list of L{BuildRequest}s that triggered me 170 @ivar build_status: the L{buildbot.status.builder.BuildStatus} that 171 collects our status 172 """ 173 174 implements(interfaces.IBuildControl) 175 176 workdir = "build" 177 build_status = None 178 reason = "changes" 179 finished = False 180 results = None 181

182 - def __init__(self, requests):

183 self.requests = requests 184 for req in self.requests: 185 req.startCount += 1 186 self.locks = [] 187 # build a source stamp 188 self.source = requests[0].mergeWith(requests[1:]) 189 self.reason = requests[0].mergeReasons(requests[1:]) 190 191 self.progress = None 192 self.currentStep = None 193 self.slaveEnvironment = {} 194 195 self.terminate = False

196

197 - def setBuilder(self, builder):

198 """ 199 Set the given builder as our builder. 200 201 @type builder: L{buildbot.process.builder.Builder} 202 """ 203 self.builder = builder

204

205 - def setLocks(self, locks):

206 self.locks = locks

207

208 - def setSlaveEnvironment(self, env):

209 self.slaveEnvironment = env

210

211 - def getSourceStamp(self):

212 return self.source

213

214 - def setProperty(self, propname, value, source):

215 """Set a property on this build. This may only be called after the 216 build has started, so that it has a BuildStatus object where the 217 properties can live.""" 218 self.build_status.setProperty(propname, value, source)

219

220 - def getProperties(self):

221 return self.build_status.getProperties()

222

223 - def getProperty(self, propname):

224 return self.build_status.getProperty(propname)

225

226 - def allChanges(self):

227 return self.source.changes

228

229 - def allFiles(self):

230 # return a list of all source files that were changed 231 files = [] 232 havedirs = 0 233 for c in self.allChanges(): 234 for f in c.files: 235 files.append(f) 236 if c.isdir: 237 havedirs = 1 238 return files

239

240 - def __repr__(self):

241 return "<Build %s>" % (self.builder.name,)

242

243 - def blamelist(self):

244 blamelist = [] 245 for c in self.allChanges(): 246 if c.who not in blamelist: 247 blamelist.append(c.who) 248 blamelist.sort() 249 return blamelist

250

251 - def changesText(self):

252 changetext = "" 253 for c in self.allChanges(): 254 changetext += "-" * 60 + "\n\n" + c.asText() + "\n" 255 # consider sorting these by number 256 return changetext

257

258 - def setStepFactories(self, step_factories):

259 """Set a list of 'step factories', which are tuples of (class, 260 kwargs), where 'class' is generally a subclass of step.BuildStep . 261 These are used to create the Steps themselves when the Build starts 262 (as opposed to when it is first created). By creating the steps 263 later, their __init__ method will have access to things like 264 build.allFiles() .""" 265 self.stepFactories = list(step_factories)

266 267 268 269 useProgress = True 270

271 - def getSlaveCommandVersion(self, command, oldversion=None):

272 return self.slavebuilder.getSlaveCommandVersion(command, oldversion)

273 - def getSlaveName(self):

274 return self.slavebuilder.slave.slavename

275

276 - def setupProperties(self):

277 props = self.getProperties() 278 279 # start with global properties from the configuration 280 buildmaster = self.builder.botmaster.parent 281 props.updateFromProperties(buildmaster.properties) 282 283 # get any properties from requests (this is the path through 284 # which schedulers will send us properties) 285 for rq in self.requests: 286 props.updateFromProperties(rq.properties) 287 288 # and finally, from the SourceStamp, which has properties via Change 289 for change in self.source.changes: 290 props.updateFromProperties(change.properties) 291 292 # now set some properties of our own, corresponding to the 293 # build itself 294 props.setProperty("buildername", self.builder.name, "Build") 295 props.setProperty("buildnumber", self.build_status.number, "Build") 296 props.setProperty("branch", self.source.branch, "Build") 297 props.setProperty("revision", self.source.revision, "Build")

298

299 - def setupSlaveBuilder(self, slavebuilder):

300 self.slavebuilder = slavebuilder 301 302 # navigate our way back to the L{buildbot.buildslave.BuildSlave} 303 # object that came from the config, and get its properties 304 buildslave_properties = slavebuilder.slave.properties 305 self.getProperties().updateFromProperties(buildslave_properties) 306 307 self.slavename = slavebuilder.slave.slavename 308 self.build_status.setSlavename(self.slavename)

309

310 - def startBuild(self, build_status, expectations, slavebuilder):

311 """This method sets up the build, then starts it by invoking the 312 first Step. It returns a Deferred which will fire when the build 313 finishes. This Deferred is guaranteed to never errback.""" 314 315 # we are taking responsibility for watching the connection to the 316 # remote. This responsibility was held by the Builder until our 317 # startBuild was called, and will not return to them until we fire 318 # the Deferred returned by this method. 319 320 log.msg("%s.startBuild" % self) 321 self.build_status = build_status 322 # now that we have a build_status, we can set properties 323 self.setupProperties() 324 self.setupSlaveBuilder(slavebuilder) 325 slavebuilder.slave.updateSlaveStatus(buildStarted=build_status) 326 327 # convert all locks into their real forms 328 lock_list = [] 329 for access in self.locks: 330 if not isinstance(access, locks.LockAccess): 331 # Buildbot 0.7.7 compability: user did not specify access 332 access = access.defaultAccess() 333 lock = self.builder.botmaster.getLockByID(access.lockid) 334 lock_list.append((lock, access)) 335 self.locks = lock_list 336 # then narrow SlaveLocks down to the right slave 337 self.locks = [(l.getLock(self.slavebuilder), la) 338 for l, la in self.locks] 339 self.remote = slavebuilder.remote 340 self.remote.notifyOnDisconnect(self.lostRemote) 341 d = self.deferred = defer.Deferred() 342 def _release_slave(res, slave, bs): 343 self.slavebuilder.buildFinished() 344 slave.updateSlaveStatus(buildFinished=bs) 345 return res

346 d.addCallback(_release_slave, self.slavebuilder.slave, build_status) 347 348 try: 349 self.setupBuild(expectations) # create .steps 350 except: 351 # the build hasn't started yet, so log the exception as a point 352 # event instead of flunking the build. TODO: associate this 353 # failure with the build instead. this involves doing 354 # self.build_status.buildStarted() from within the exception 355 # handler 356 log.msg("Build.setupBuild failed") 357 log.err(Failure()) 358 self.builder.builder_status.addPointEvent(["setupBuild", 359 "exception"]) 360 self.finished = True 361 self.results = FAILURE 362 self.deferred = None 363 d.callback(self) 364 return d 365 366 self.acquireLocks().addCallback(self._startBuild_2) 367 return d

368

369 - def acquireLocks(self, res=None):

370 log.msg("acquireLocks(step %s, locks %s)" % (self, self.locks)) 371 if not self.locks: 372 return defer.succeed(None) 373 for lock, access in self.locks: 374 if not lock.isAvailable(access): 375 log.msg("Build %s waiting for lock %s" % (self, lock)) 376 d = lock.waitUntilMaybeAvailable(self, access) 377 d.addCallback(self.acquireLocks) 378 return d 379 # all locks are available, claim them all 380 for lock, access in self.locks: 381 lock.claim(self, access) 382 return defer.succeed(None)

383

384 - def _startBuild_2(self, res):

385 self.build_status.buildStarted(self) 386 self.startNextStep()

387

388 - def setupBuild(self, expectations):

389 # create the actual BuildSteps. If there are any name collisions, we 390 # add a count to the loser until it is unique. 391 self.steps = [] 392 self.stepStatuses = {} 393 stepnames = {} 394 sps = [] 395 396 for factory, args in self.stepFactories: 397 args = args.copy() 398 try: 399 step = factory(**args) 400 except: 401 log.msg("error while creating step, factory=%s, args=%s" 402 % (factory, args)) 403 raise 404 step.setBuild(self) 405 step.setBuildSlave(self.slavebuilder.slave) 406 step.setDefaultWorkdir(self.workdir) 407 name = step.name 408 if stepnames.has_key(name): 409 count = stepnames[name] 410 count += 1 411 stepnames[name] = count 412 name = step.name + "_%d" % count 413 else: 414 stepnames[name] = 0 415 step.name = name 416 self.steps.append(step) 417 418 # tell the BuildStatus about the step. This will create a 419 # BuildStepStatus and bind it to the Step. 420 step_status = self.build_status.addStepWithName(name) 421 step.setStepStatus(step_status) 422 423 sp = None 424 if self.useProgress: 425 # XXX: maybe bail if step.progressMetrics is empty? or skip 426 # progress for that one step (i.e. "it is fast"), or have a 427 # separate "variable" flag that makes us bail on progress 428 # tracking 429 sp = step.setupProgress() 430 if sp: 431 sps.append(sp) 432 433 # Create a buildbot.status.progress.BuildProgress object. This is 434 # called once at startup to figure out how to build the long-term 435 # Expectations object, and again at the start of each build to get a 436 # fresh BuildProgress object to track progress for that individual 437 # build. TODO: revisit at-startup call 438 439 if self.useProgress: 440 self.progress = BuildProgress(sps) 441 if self.progress and expectations: 442 self.progress.setExpectationsFrom(expectations) 443 444 # we are now ready to set up our BuildStatus. 445 self.build_status.setSourceStamp(self.source) 446 self.build_status.setRequests([req.status for req in self.requests]) 447 self.build_status.setReason(self.reason) 448 self.build_status.setBlamelist(self.blamelist()) 449 self.build_status.setProgress(self.progress) 450 451 # gather owners from build requests 452 owners = [r.properties['owner'] for r in self.requests 453 if r.properties.has_key('owner')] 454 if owners: self.setProperty('owners', owners, self.reason) 455 456 self.results = [] # list of FAILURE, SUCCESS, WARNINGS, SKIPPED 457 self.result = SUCCESS # overall result, may downgrade after each step 458 self.text = [] # list of text string lists (text2)

459

460 - def getNextStep(self):

461 """This method is called to obtain the next BuildStep for this build. 462 When it returns None (or raises a StopIteration exception), the build 463 is complete.""" 464 if not self.steps: 465 return None 466 if self.terminate: 467 while True: 468 s = self.steps.pop(0) 469 if s.alwaysRun: 470 return s 471 if not self.steps: 472 return None 473 else: 474 return self.steps.pop(0)

475

476 - def startNextStep(self):

477 try: 478 s = self.getNextStep() 479 except StopIteration: 480 s = None 481 if not s: 482 return self.allStepsDone() 483 self.currentStep = s 484 d = defer.maybeDeferred(s.startStep, self.remote) 485 d.addCallback(self._stepDone, s) 486 d.addErrback(self.buildException)

487

488 - def _stepDone(self, results, step):

489 self.currentStep = None 490 if self.finished: 491 return # build was interrupted, don't keep building 492 terminate = self.stepDone(results, step) # interpret/merge results 493 if terminate: 494 self.terminate = True 495 return self.startNextStep()

496

497 - def stepDone(self, result, step):

498 """This method is called when the BuildStep completes. It is passed a 499 status object from the BuildStep and is responsible for merging the 500 Step's results into those of the overall Build.""" 501 502 terminate = False 503 text = None 504 if type(result) == types.TupleType: 505 result, text = result 506 assert type(result) == type(SUCCESS) 507 log.msg(" step '%s' complete: %s" % (step.name, Results[result])) 508 self.results.append(result) 509 if text: 510 self.text.extend(text) 511 if not self.remote: 512 terminate = True 513 if result == FAILURE: 514 if step.warnOnFailure: 515 if self.result != FAILURE: 516 self.result = WARNINGS 517 if step.flunkOnFailure: 518 self.result = FAILURE 519 if step.haltOnFailure: 520 terminate = True 521 elif result == WARNINGS: 522 if step.warnOnWarnings: 523 if self.result != FAILURE: 524 self.result = WARNINGS 525 if step.flunkOnWarnings: 526 self.result = FAILURE 527 elif result == EXCEPTION: 528 self.result = EXCEPTION 529 terminate = True 530 return terminate

531

532 - def lostRemote(self, remote=None):

533 # the slave went away. There are several possible reasons for this, 534 # and they aren't necessarily fatal. For now, kill the build, but 535 # TODO: see if we can resume the build when it reconnects. 536 log.msg("%s.lostRemote" % self) 537 self.remote = None 538 if self.currentStep: 539 # this should cause the step to finish. 540 log.msg(" stopping currentStep", self.currentStep) 541 self.currentStep.interrupt(Failure(error.ConnectionLost()))

542

543 - def stopBuild(self, reason="<no reason given>"):

544 # the idea here is to let the user cancel a build because, e.g., 545 # they realized they committed a bug and they don't want to waste 546 # the time building something that they know will fail. Another 547 # reason might be to abandon a stuck build. We want to mark the 548 # build as failed quickly rather than waiting for the slave's 549 # timeout to kill it on its own. 550 551 log.msg(" %s: stopping build: %s" % (self, reason)) 552 if self.finished: 553 return 554 # TODO: include 'reason' in this point event 555 self.builder.builder_status.addPointEvent(['interrupt']) 556 self.currentStep.interrupt(reason) 557 if 0: 558 # TODO: maybe let its deferred do buildFinished 559 if self.currentStep and self.currentStep.progress: 560 # XXX: really .fail or something 561 self.currentStep.progress.finish() 562 text = ["stopped", reason] 563 self.buildFinished(text, FAILURE)

564

565 - def allStepsDone(self):

566 if self.result == FAILURE: 567 text = ["failed"] 568 elif self.result == WARNINGS: 569 text = ["warnings"] 570 elif self.result == EXCEPTION: 571 text = ["exception"] 572 else: 573 text = ["build", "successful"] 574 text.extend(self.text) 575 return self.buildFinished(text, self.result)

576

577 - def buildException(self, why):

578 log.msg("%s.buildException" % self) 579 log.err(why) 580 self.buildFinished(["build", "exception"], FAILURE)

581

582 - def buildFinished(self, text, results):

583 """This method must be called when the last Step has completed. It 584 marks the Build as complete and returns the Builder to the 'idle' 585 state. 586 587 It takes two arguments which describe the overall build status: 588 text, results. 'results' is one of SUCCESS, WARNINGS, or FAILURE. 589 590 If 'results' is SUCCESS or WARNINGS, we will permit any dependant 591 builds to start. If it is 'FAILURE', those builds will be 592 abandoned.""" 593 594 self.finished = True 595 if self.remote: 596 self.remote.dontNotifyOnDisconnect(self.lostRemote) 597 self.results = results 598 599 log.msg(" %s: build finished" % self) 600 self.build_status.setText(text) 601 self.build_status.setResults(results) 602 self.build_status.buildFinished() 603 if self.progress and results == SUCCESS: 604 # XXX: also test a 'timing consistent' flag? 605 log.msg(" setting expectations for next time") 606 self.builder.setExpectations(self.progress) 607 reactor.callLater(0, self.releaseLocks) 608 self.deferred.callback(self) 609 self.deferred = None

610

611 - def releaseLocks(self):

612 log.msg("releaseLocks(%s): %s" % (self, self.locks)) 613 for lock, access in self.locks: 614 lock.release(self, access)

615 616 # IBuildControl 617

618 - def getStatus(self):

619 return self.build_status

620 621 # stopBuild is defined earlier 622

Source Code for Module buildbot.process.base