buildbot.process.build

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 17 import types 18 19 from zope.interface import implements 20 from twisted.python import log, components 21 from twisted.python.failure import Failure 22 from twisted.internet import reactor, defer, error 23 24 from buildbot import interfaces, locks 25 from buildbot.status.results import SUCCESS, WARNINGS, FAILURE, EXCEPTION, \ 26 RETRY, SKIPPED, worst_status 27 from buildbot.status.builder import Results 28 from buildbot.status.progress import BuildProgress 29 from buildbot.process import metrics, properties 30 31

32 -class Build(properties.PropertiesMixin):

33 """I represent a single build by a single slave. Specialized Builders can 34 use subclasses of Build to hold status information unique to those build 35 processes. 36 37 I control B{how} the build proceeds. The actual build is broken up into a 38 series of steps, saved in the .buildSteps[] array as a list of 39 L{buildbot.process.step.BuildStep} objects. Each step is a single remote 40 command, possibly a shell command. 41 42 During the build, I put status information into my C{BuildStatus} 43 gatherer. 44 45 After the build, I go away. 46 47 I can be used by a factory by setting buildClass on 48 L{buildbot.process.factory.BuildFactory} 49 50 @ivar requests: the list of L{BuildRequest}s that triggered me 51 @ivar build_status: the L{buildbot.status.build.BuildStatus} that 52 collects our status 53 """ 54 55 implements(interfaces.IBuildControl) 56 57 workdir = "build" 58 build_status = None 59 reason = "changes" 60 finished = False 61 results = None 62 stopped = False 63 set_runtime_properties = True 64

65 - def __init__(self, requests):

66 self.requests = requests 67 self.locks = [] 68 69 # build a source stamp 70 self.source = requests[0].mergeWith(requests[1:]) 71 self.reason = requests[0].mergeReasons(requests[1:]) 72 73 self.progress = None 74 self.currentStep = None 75 self.slaveEnvironment = {} 76 77 self.terminate = False 78 79 self._acquiringLock = None

80

81 - def setBuilder(self, builder):

82 """ 83 Set the given builder as our builder. 84 85 @type builder: L{buildbot.process.builder.Builder} 86 """ 87 self.builder = builder

88

89 - def setLocks(self, locks):

90 self.locks = locks

91

92 - def setSlaveEnvironment(self, env):

93 self.slaveEnvironment = env

94

95 - def getSourceStamp(self, repository=None):

96 if repository is None: 97 return self.source 98 assert (repository in self.sources) 99 return self.sources[repository]

100

101 - def allChanges(self):

102 return self.source.changes

103

104 - def allFiles(self):

105 # return a list of all source files that were changed 106 files = [] 107 for c in self.allChanges(): 108 for f in c.files: 109 files.append(f) 110 return files

111

112 - def __repr__(self):

113 return "<Build %s>" % (self.builder.name,)

114

115 - def blamelist(self):

116 blamelist = [] 117 for c in self.allChanges(): 118 if c.who not in blamelist: 119 blamelist.append(c.who) 120 if self.source.patch_info: #Add patch author to blamelist 121 blamelist.append(self.source.patch_info[0]) 122 blamelist.sort() 123 return blamelist

124

125 - def changesText(self):

126 changetext = "" 127 for c in self.allChanges(): 128 changetext += "-" * 60 + "\n\n" + c.asText() + "\n" 129 # consider sorting these by number 130 return changetext

131

132 - def setStepFactories(self, step_factories):

133 """Set a list of 'step factories', which are tuples of (class, 134 kwargs), where 'class' is generally a subclass of step.BuildStep . 135 These are used to create the Steps themselves when the Build starts 136 (as opposed to when it is first created). By creating the steps 137 later, their __init__ method will have access to things like 138 build.allFiles() .""" 139 self.stepFactories = list(step_factories)

140 141 useProgress = True 142

143 - def getSlaveCommandVersion(self, command, oldversion=None):

144 return self.slavebuilder.getSlaveCommandVersion(command, oldversion)

145 - def getSlaveName(self):

146 return self.slavebuilder.slave.slavename

147

148 - def setupProperties(self):

149 props = interfaces.IProperties(self) 150 151 # give the properties a reference back to this build 152 props.build = self 153 154 # start with global properties from the configuration 155 buildmaster = self.builder.botmaster.parent 156 props.updateFromProperties(buildmaster.config.properties) 157 158 # from the SourceStamp, which has properties via Change 159 for change in self.source.changes: 160 props.updateFromProperties(change.properties) 161 162 # and finally, get any properties from requests (this is the path 163 # through which schedulers will send us properties) 164 for rq in self.requests: 165 props.updateFromProperties(rq.properties) 166 167 # now set some properties of our own, corresponding to the 168 # build itself 169 props.setProperty("buildnumber", self.build_status.number, "Build") 170 props.setProperty("branch", self.source.branch, "Build") 171 props.setProperty("revision", self.source.revision, "Build") 172 props.setProperty("repository", self.source.repository, "Build") 173 props.setProperty("project", self.source.project, "Build") 174 self.builder.setupProperties(props)

175

176 - def setupSlaveBuilder(self, slavebuilder):

177 self.slavebuilder = slavebuilder 178 179 # navigate our way back to the L{buildbot.buildslave.BuildSlave} 180 # object that came from the config, and get its properties 181 buildslave_properties = slavebuilder.slave.properties 182 self.getProperties().updateFromProperties(buildslave_properties) 183 if slavebuilder.slave.slave_basedir: 184 self.setProperty("workdir", 185 slavebuilder.slave.path_module.join( 186 slavebuilder.slave.slave_basedir, 187 self.builder.config.slavebuilddir), 188 "slave") 189 190 self.slavename = slavebuilder.slave.slavename 191 self.build_status.setSlavename(self.slavename)

192

193 - def startBuild(self, build_status, expectations, slavebuilder):

194 """This method sets up the build, then starts it by invoking the 195 first Step. It returns a Deferred which will fire when the build 196 finishes. This Deferred is guaranteed to never errback.""" 197 198 # we are taking responsibility for watching the connection to the 199 # remote. This responsibility was held by the Builder until our 200 # startBuild was called, and will not return to them until we fire 201 # the Deferred returned by this method. 202 203 log.msg("%s.startBuild" % self) 204 self.build_status = build_status 205 # now that we have a build_status, we can set properties 206 self.setupProperties() 207 self.setupSlaveBuilder(slavebuilder) 208 slavebuilder.slave.updateSlaveStatus(buildStarted=build_status) 209 210 # convert all locks into their real forms 211 lock_list = [] 212 for access in self.locks: 213 if not isinstance(access, locks.LockAccess): 214 # Buildbot 0.7.7 compability: user did not specify access 215 access = access.defaultAccess() 216 lock = self.builder.botmaster.getLockByID(access.lockid) 217 lock_list.append((lock, access)) 218 self.locks = lock_list 219 # then narrow SlaveLocks down to the right slave 220 self.locks = [(l.getLock(self.slavebuilder), la) 221 for l, la in self.locks] 222 self.remote = slavebuilder.remote 223 self.remote.notifyOnDisconnect(self.lostRemote) 224 225 metrics.MetricCountEvent.log('active_builds', 1) 226 227 d = self.deferred = defer.Deferred() 228 def _uncount_build(res): 229 metrics.MetricCountEvent.log('active_builds', -1) 230 return res

231 d.addBoth(_uncount_build) 232 233 def _release_slave(res, slave, bs): 234 self.slavebuilder.buildFinished() 235 slave.updateSlaveStatus(buildFinished=bs) 236 return res

237 d.addCallback(_release_slave, self.slavebuilder.slave, build_status) 238 239 try: 240 self.setupBuild(expectations) # create .steps 241 except: 242 # the build hasn't started yet, so log the exception as a point 243 # event instead of flunking the build. 244 # TODO: associate this failure with the build instead. 245 # this involves doing 246 # self.build_status.buildStarted() from within the exception 247 # handler 248 log.msg("Build.setupBuild failed") 249 log.err(Failure()) 250 self.builder.builder_status.addPointEvent(["setupBuild", 251 "exception"]) 252 self.finished = True 253 self.results = FAILURE 254 self.deferred = None 255 d.callback(self) 256 return d 257 258 self.build_status.buildStarted(self) 259 self.acquireLocks().addCallback(self._startBuild_2) 260 return d 261

262 - def acquireLocks(self, res=None):

263 self._acquiringLock = None 264 if not self.locks: 265 return defer.succeed(None) 266 if self.stopped: 267 return defer.succeed(None) 268 log.msg("acquireLocks(build %s, locks %s)" % (self, self.locks)) 269 for lock, access in self.locks: 270 if not lock.isAvailable(access): 271 log.msg("Build %s waiting for lock %s" % (self, lock)) 272 d = lock.waitUntilMaybeAvailable(self, access) 273 d.addCallback(self.acquireLocks) 274 self._acquiringLock = (lock, access, d) 275 return d 276 # all locks are available, claim them all 277 for lock, access in self.locks: 278 lock.claim(self, access) 279 return defer.succeed(None)

280

281 - def _startBuild_2(self, res):

282 self.startNextStep()

283

284 - def setupBuild(self, expectations):

285 # create the actual BuildSteps. If there are any name collisions, we 286 # add a count to the loser until it is unique. 287 self.steps = [] 288 self.stepStatuses = {} 289 stepnames = {} 290 sps = [] 291 292 for factory, args in self.stepFactories: 293 args = args.copy() 294 try: 295 step = factory(**args) 296 except: 297 log.msg("error while creating step, factory=%s, args=%s" 298 % (factory, args)) 299 raise 300 step.setBuild(self) 301 step.setBuildSlave(self.slavebuilder.slave) 302 if callable (self.workdir): 303 step.setDefaultWorkdir (self.workdir (self.source)) 304 else: 305 step.setDefaultWorkdir (self.workdir) 306 name = step.name 307 if stepnames.has_key(name): 308 count = stepnames[name] 309 count += 1 310 stepnames[name] = count 311 name = step.name + "_%d" % count 312 else: 313 stepnames[name] = 0 314 step.name = name 315 self.steps.append(step) 316 317 # tell the BuildStatus about the step. This will create a 318 # BuildStepStatus and bind it to the Step. 319 step_status = self.build_status.addStepWithName(name) 320 step.setStepStatus(step_status) 321 322 sp = None 323 if self.useProgress: 324 # XXX: maybe bail if step.progressMetrics is empty? or skip 325 # progress for that one step (i.e. "it is fast"), or have a 326 # separate "variable" flag that makes us bail on progress 327 # tracking 328 sp = step.setupProgress() 329 if sp: 330 sps.append(sp) 331 332 # Create a buildbot.status.progress.BuildProgress object. This is 333 # called once at startup to figure out how to build the long-term 334 # Expectations object, and again at the start of each build to get a 335 # fresh BuildProgress object to track progress for that individual 336 # build. TODO: revisit at-startup call 337 338 if self.useProgress: 339 self.progress = BuildProgress(sps) 340 if self.progress and expectations: 341 self.progress.setExpectationsFrom(expectations) 342 343 # we are now ready to set up our BuildStatus. 344 self.build_status.setSourceStamp(self.source) 345 self.build_status.setReason(self.reason) 346 self.build_status.setBlamelist(self.blamelist()) 347 self.build_status.setProgress(self.progress) 348 349 # gather owners from build requests 350 owners = [r.properties['owner'] for r in self.requests 351 if r.properties.has_key('owner')] 352 if owners: self.setProperty('owners', owners, self.reason) 353 354 self.results = [] # list of FAILURE, SUCCESS, WARNINGS, SKIPPED 355 self.result = SUCCESS # overall result, may downgrade after each step 356 self.text = [] # list of text string lists (text2)

357

358 - def getNextStep(self):

359 """This method is called to obtain the next BuildStep for this build. 360 When it returns None (or raises a StopIteration exception), the build 361 is complete.""" 362 if not self.steps: 363 return None 364 if not self.remote: 365 return None 366 if self.terminate or self.stopped: 367 # Run any remaining alwaysRun steps, and skip over the others 368 while True: 369 s = self.steps.pop(0) 370 if s.alwaysRun: 371 return s 372 if not self.steps: 373 return None 374 else: 375 return self.steps.pop(0)

376

377 - def startNextStep(self):

378 try: 379 s = self.getNextStep() 380 except StopIteration: 381 s = None 382 if not s: 383 return self.allStepsDone() 384 self.currentStep = s 385 d = defer.maybeDeferred(s.startStep, self.remote) 386 d.addCallback(self._stepDone, s) 387 d.addErrback(self.buildException)

388

389 - def _stepDone(self, results, step):

390 self.currentStep = None 391 if self.finished: 392 return # build was interrupted, don't keep building 393 terminate = self.stepDone(results, step) # interpret/merge results 394 if terminate: 395 self.terminate = True 396 return self.startNextStep()

397

398 - def stepDone(self, result, step):

399 """This method is called when the BuildStep completes. It is passed a 400 status object from the BuildStep and is responsible for merging the 401 Step's results into those of the overall Build.""" 402 403 terminate = False 404 text = None 405 if type(result) == types.TupleType: 406 result, text = result 407 assert type(result) == type(SUCCESS) 408 log.msg(" step '%s' complete: %s" % (step.name, Results[result])) 409 self.results.append(result) 410 if text: 411 self.text.extend(text) 412 if not self.remote: 413 terminate = True 414 415 possible_overall_result = result 416 if result == FAILURE: 417 if not step.flunkOnFailure: 418 possible_overall_result = SUCCESS 419 if step.warnOnFailure: 420 possible_overall_result = WARNINGS 421 if step.flunkOnFailure: 422 possible_overall_result = FAILURE 423 if step.haltOnFailure: 424 terminate = True 425 elif result == WARNINGS: 426 if not step.warnOnWarnings: 427 possible_overall_result = SUCCESS 428 else: 429 possible_overall_result = WARNINGS 430 if step.flunkOnWarnings: 431 possible_overall_result = FAILURE 432 elif result in (EXCEPTION, RETRY): 433 terminate = True 434 435 # if we skipped this step, then don't adjust the build status 436 if result != SKIPPED: 437 self.result = worst_status(self.result, possible_overall_result) 438 439 return terminate

440

441 - def lostRemote(self, remote=None):

442 # the slave went away. There are several possible reasons for this, 443 # and they aren't necessarily fatal. For now, kill the build, but 444 # TODO: see if we can resume the build when it reconnects. 445 log.msg("%s.lostRemote" % self) 446 self.remote = None 447 if self.currentStep: 448 # this should cause the step to finish. 449 log.msg(" stopping currentStep", self.currentStep) 450 self.currentStep.interrupt(Failure(error.ConnectionLost())) 451 else: 452 self.result = RETRY 453 self.text = ["lost", "remote"] 454 self.stopped = True 455 if self._acquiringLock: 456 lock, access, d = self._acquiringLock 457 lock.stopWaitingUntilAvailable(self, access, d) 458 d.callback(None)

459

460 - def stopBuild(self, reason="<no reason given>"):

461 # the idea here is to let the user cancel a build because, e.g., 462 # they realized they committed a bug and they don't want to waste 463 # the time building something that they know will fail. Another 464 # reason might be to abandon a stuck build. We want to mark the 465 # build as failed quickly rather than waiting for the slave's 466 # timeout to kill it on its own. 467 468 log.msg(" %s: stopping build: %s" % (self, reason)) 469 if self.finished: 470 return 471 # TODO: include 'reason' in this point event 472 self.builder.builder_status.addPointEvent(['interrupt']) 473 self.stopped = True 474 if self.currentStep: 475 self.currentStep.interrupt(reason) 476 477 self.result = EXCEPTION 478 479 if self._acquiringLock: 480 lock, access, d = self._acquiringLock 481 lock.stopWaitingUntilAvailable(self, access, d) 482 d.callback(None)

483

484 - def allStepsDone(self):

485 if self.result == FAILURE: 486 text = ["failed"] 487 elif self.result == WARNINGS: 488 text = ["warnings"] 489 elif self.result == EXCEPTION: 490 text = ["exception"] 491 elif self.result == RETRY: 492 text = ["retry"] 493 else: 494 text = ["build", "successful"] 495 text.extend(self.text) 496 return self.buildFinished(text, self.result)

497

498 - def buildException(self, why):

499 log.msg("%s.buildException" % self) 500 log.err(why) 501 # try to finish the build, but since we've already faced an exception, 502 # this may not work well. 503 try: 504 self.buildFinished(["build", "exception"], EXCEPTION) 505 except: 506 log.err(Failure(), 'while finishing a build with an exception')

507

508 - def buildFinished(self, text, results):

509 """This method must be called when the last Step has completed. It 510 marks the Build as complete and returns the Builder to the 'idle' 511 state. 512 513 It takes two arguments which describe the overall build status: 514 text, results. 'results' is one of SUCCESS, WARNINGS, or FAILURE. 515 516 If 'results' is SUCCESS or WARNINGS, we will permit any dependant 517 builds to start. If it is 'FAILURE', those builds will be 518 abandoned.""" 519 520 self.finished = True 521 if self.remote: 522 self.remote.dontNotifyOnDisconnect(self.lostRemote) 523 self.remote = None 524 self.results = results 525 526 log.msg(" %s: build finished" % self) 527 self.build_status.setText(text) 528 self.build_status.setResults(results) 529 self.build_status.buildFinished() 530 if self.progress and results == SUCCESS: 531 # XXX: also test a 'timing consistent' flag? 532 log.msg(" setting expectations for next time") 533 self.builder.setExpectations(self.progress) 534 reactor.callLater(0, self.releaseLocks) 535 self.deferred.callback(self) 536 self.deferred = None

537

538 - def releaseLocks(self):

539 if self.locks: 540 log.msg("releaseLocks(%s): %s" % (self, self.locks)) 541 for lock, access in self.locks: 542 if lock.isOwner(self, access): 543 lock.release(self, access) 544 else: 545 # This should only happen if we've been interrupted 546 assert self.stopped

547 548 # IBuildControl 549

550 - def getStatus(self):

551 return self.build_status

552 553 # stopBuild is defined earlier 554 555 components.registerAdapter( 556 lambda build : interfaces.IProperties(build.build_status), 557 Build, interfaces.IProperties) 558

Source Code for Module buildbot.process.build