buildbot.process.build

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 17 import types 18 19 from zope.interface import implements 20 from twisted.python import log, components 21 from twisted.python.failure import Failure 22 from twisted.internet import reactor, defer, error 23 24 from buildbot import interfaces, locks 25 from buildbot.status.results import SUCCESS, WARNINGS, FAILURE, EXCEPTION, \ 26 RETRY, SKIPPED, worst_status 27 from buildbot.status.builder import Results 28 from buildbot.status.progress import BuildProgress 29 from buildbot.process import metrics, properties 30 31

32 -class Build(properties.PropertiesMixin):

33 """I represent a single build by a single slave. Specialized Builders can 34 use subclasses of Build to hold status information unique to those build 35 processes. 36 37 I control B{how} the build proceeds. The actual build is broken up into a 38 series of steps, saved in the .buildSteps[] array as a list of 39 L{buildbot.process.step.BuildStep} objects. Each step is a single remote 40 command, possibly a shell command. 41 42 During the build, I put status information into my C{BuildStatus} 43 gatherer. 44 45 After the build, I go away. 46 47 I can be used by a factory by setting buildClass on 48 L{buildbot.process.factory.BuildFactory} 49 50 @ivar requests: the list of L{BuildRequest}s that triggered me 51 @ivar build_status: the L{buildbot.status.build.BuildStatus} that 52 collects our status 53 """ 54 55 implements(interfaces.IBuildControl) 56 57 workdir = "build" 58 build_status = None 59 reason = "changes" 60 finished = False 61 results = None 62 stopped = False 63 set_runtime_properties = True 64

65 - def __init__(self, requests):

66 self.requests = requests 67 self.locks = [] 68 69 # build a source stamp 70 self.source = requests[0].mergeWith(requests[1:]) 71 self.reason = requests[0].mergeReasons(requests[1:]) 72 73 self.progress = None 74 self.currentStep = None 75 self.slaveEnvironment = {} 76 77 self.terminate = False 78 79 self._acquiringLock = None

80

81 - def setBuilder(self, builder):

82 """ 83 Set the given builder as our builder. 84 85 @type builder: L{buildbot.process.builder.Builder} 86 """ 87 self.builder = builder

88

89 - def setLocks(self, locks):

90 self.locks = locks

91

92 - def setSlaveEnvironment(self, env):

93 self.slaveEnvironment = env

94

95 - def getSourceStamp(self):

96 return self.source

97

98 - def allChanges(self):

99 return self.source.changes

100

101 - def allFiles(self):

102 # return a list of all source files that were changed 103 files = [] 104 for c in self.allChanges(): 105 for f in c.files: 106 files.append(f) 107 return files

108

109 - def __repr__(self):

110 return "<Build %s>" % (self.builder.name,)

111

112 - def blamelist(self):

113 blamelist = [] 114 for c in self.allChanges(): 115 if c.who not in blamelist: 116 blamelist.append(c.who) 117 if self.source.patch_info: #Add patch author to blamelist 118 blamelist.append(self.source.patch_info[0]) 119 blamelist.sort() 120 return blamelist

121

122 - def changesText(self):

123 changetext = "" 124 for c in self.allChanges(): 125 changetext += "-" * 60 + "\n\n" + c.asText() + "\n" 126 # consider sorting these by number 127 return changetext

128

129 - def setStepFactories(self, step_factories):

130 """Set a list of 'step factories', which are tuples of (class, 131 kwargs), where 'class' is generally a subclass of step.BuildStep . 132 These are used to create the Steps themselves when the Build starts 133 (as opposed to when it is first created). By creating the steps 134 later, their __init__ method will have access to things like 135 build.allFiles() .""" 136 self.stepFactories = list(step_factories)

137 138 useProgress = True 139

140 - def getSlaveCommandVersion(self, command, oldversion=None):

141 return self.slavebuilder.getSlaveCommandVersion(command, oldversion)

142 - def getSlaveName(self):

143 return self.slavebuilder.slave.slavename

144

145 - def setupProperties(self):

146 props = interfaces.IProperties(self) 147 148 # give the properties a reference back to this build 149 props.build = self 150 151 # start with global properties from the configuration 152 buildmaster = self.builder.botmaster.parent 153 props.updateFromProperties(buildmaster.properties) 154 155 # from the SourceStamp, which has properties via Change 156 for change in self.source.changes: 157 props.updateFromProperties(change.properties) 158 159 # and finally, get any properties from requests (this is the path 160 # through which schedulers will send us properties) 161 for rq in self.requests: 162 props.updateFromProperties(rq.properties) 163 164 # now set some properties of our own, corresponding to the 165 # build itself 166 props.setProperty("buildnumber", self.build_status.number, "Build") 167 props.setProperty("branch", self.source.branch, "Build") 168 props.setProperty("revision", self.source.revision, "Build") 169 props.setProperty("repository", self.source.repository, "Build") 170 props.setProperty("project", self.source.project, "Build") 171 self.builder.setupProperties(props)

172

173 - def setupSlaveBuilder(self, slavebuilder):

174 self.slavebuilder = slavebuilder 175 176 # navigate our way back to the L{buildbot.buildslave.BuildSlave} 177 # object that came from the config, and get its properties 178 buildslave_properties = slavebuilder.slave.properties 179 self.getProperties().updateFromProperties(buildslave_properties) 180 if slavebuilder.slave.slave_basedir: 181 self.setProperty("workdir", slavebuilder.slave.path_module.join(slavebuilder.slave.slave_basedir, self.builder.slavebuilddir), "slave") 182 183 self.slavename = slavebuilder.slave.slavename 184 self.build_status.setSlavename(self.slavename)

185

186 - def startBuild(self, build_status, expectations, slavebuilder):

187 """This method sets up the build, then starts it by invoking the 188 first Step. It returns a Deferred which will fire when the build 189 finishes. This Deferred is guaranteed to never errback.""" 190 191 # we are taking responsibility for watching the connection to the 192 # remote. This responsibility was held by the Builder until our 193 # startBuild was called, and will not return to them until we fire 194 # the Deferred returned by this method. 195 196 log.msg("%s.startBuild" % self) 197 self.build_status = build_status 198 # now that we have a build_status, we can set properties 199 self.setupProperties() 200 self.setupSlaveBuilder(slavebuilder) 201 slavebuilder.slave.updateSlaveStatus(buildStarted=build_status) 202 203 # convert all locks into their real forms 204 lock_list = [] 205 for access in self.locks: 206 if not isinstance(access, locks.LockAccess): 207 # Buildbot 0.7.7 compability: user did not specify access 208 access = access.defaultAccess() 209 lock = self.builder.botmaster.getLockByID(access.lockid) 210 lock_list.append((lock, access)) 211 self.locks = lock_list 212 # then narrow SlaveLocks down to the right slave 213 self.locks = [(l.getLock(self.slavebuilder), la) 214 for l, la in self.locks] 215 self.remote = slavebuilder.remote 216 self.remote.notifyOnDisconnect(self.lostRemote) 217 218 metrics.MetricCountEvent.log('active_builds', 1) 219 220 d = self.deferred = defer.Deferred() 221 def _uncount_build(res): 222 metrics.MetricCountEvent.log('active_builds', -1) 223 return res

224 d.addBoth(_uncount_build) 225 226 def _release_slave(res, slave, bs): 227 self.slavebuilder.buildFinished() 228 slave.updateSlaveStatus(buildFinished=bs) 229 return res

230 d.addCallback(_release_slave, self.slavebuilder.slave, build_status) 231 232 try: 233 self.setupBuild(expectations) # create .steps 234 except: 235 # the build hasn't started yet, so log the exception as a point 236 # event instead of flunking the build. 237 # TODO: associate this failure with the build instead. 238 # this involves doing 239 # self.build_status.buildStarted() from within the exception 240 # handler 241 log.msg("Build.setupBuild failed") 242 log.err(Failure()) 243 self.builder.builder_status.addPointEvent(["setupBuild", 244 "exception"]) 245 self.finished = True 246 self.results = FAILURE 247 self.deferred = None 248 d.callback(self) 249 return d 250 251 self.build_status.buildStarted(self) 252 self.acquireLocks().addCallback(self._startBuild_2) 253 return d 254

255 - def acquireLocks(self, res=None):

256 self._acquiringLock = None 257 if not self.locks: 258 return defer.succeed(None) 259 if self.stopped: 260 return defer.succeed(None) 261 log.msg("acquireLocks(build %s, locks %s)" % (self, self.locks)) 262 for lock, access in self.locks: 263 if not lock.isAvailable(access): 264 log.msg("Build %s waiting for lock %s" % (self, lock)) 265 d = lock.waitUntilMaybeAvailable(self, access) 266 d.addCallback(self.acquireLocks) 267 self._acquiringLock = (lock, access, d) 268 return d 269 # all locks are available, claim them all 270 for lock, access in self.locks: 271 lock.claim(self, access) 272 return defer.succeed(None)

273

274 - def _startBuild_2(self, res):

275 self.startNextStep()

276

277 - def setupBuild(self, expectations):

278 # create the actual BuildSteps. If there are any name collisions, we 279 # add a count to the loser until it is unique. 280 self.steps = [] 281 self.stepStatuses = {} 282 stepnames = {} 283 sps = [] 284 285 for factory, args in self.stepFactories: 286 args = args.copy() 287 try: 288 step = factory(**args) 289 except: 290 log.msg("error while creating step, factory=%s, args=%s" 291 % (factory, args)) 292 raise 293 step.setBuild(self) 294 step.setBuildSlave(self.slavebuilder.slave) 295 if callable (self.workdir): 296 step.setDefaultWorkdir (self.workdir (self.source)) 297 else: 298 step.setDefaultWorkdir (self.workdir) 299 name = step.name 300 if stepnames.has_key(name): 301 count = stepnames[name] 302 count += 1 303 stepnames[name] = count 304 name = step.name + "_%d" % count 305 else: 306 stepnames[name] = 0 307 step.name = name 308 self.steps.append(step) 309 310 # tell the BuildStatus about the step. This will create a 311 # BuildStepStatus and bind it to the Step. 312 step_status = self.build_status.addStepWithName(name) 313 step.setStepStatus(step_status) 314 315 sp = None 316 if self.useProgress: 317 # XXX: maybe bail if step.progressMetrics is empty? or skip 318 # progress for that one step (i.e. "it is fast"), or have a 319 # separate "variable" flag that makes us bail on progress 320 # tracking 321 sp = step.setupProgress() 322 if sp: 323 sps.append(sp) 324 325 # Create a buildbot.status.progress.BuildProgress object. This is 326 # called once at startup to figure out how to build the long-term 327 # Expectations object, and again at the start of each build to get a 328 # fresh BuildProgress object to track progress for that individual 329 # build. TODO: revisit at-startup call 330 331 if self.useProgress: 332 self.progress = BuildProgress(sps) 333 if self.progress and expectations: 334 self.progress.setExpectationsFrom(expectations) 335 336 # we are now ready to set up our BuildStatus. 337 self.build_status.setSourceStamp(self.source) 338 self.build_status.setReason(self.reason) 339 self.build_status.setBlamelist(self.blamelist()) 340 self.build_status.setProgress(self.progress) 341 342 # gather owners from build requests 343 owners = [r.properties['owner'] for r in self.requests 344 if r.properties.has_key('owner')] 345 if owners: self.setProperty('owners', owners, self.reason) 346 347 self.results = [] # list of FAILURE, SUCCESS, WARNINGS, SKIPPED 348 self.result = SUCCESS # overall result, may downgrade after each step 349 self.text = [] # list of text string lists (text2)

350

351 - def getNextStep(self):

352 """This method is called to obtain the next BuildStep for this build. 353 When it returns None (or raises a StopIteration exception), the build 354 is complete.""" 355 if not self.steps: 356 return None 357 if not self.remote: 358 return None 359 if self.terminate or self.stopped: 360 # Run any remaining alwaysRun steps, and skip over the others 361 while True: 362 s = self.steps.pop(0) 363 if s.alwaysRun: 364 return s 365 if not self.steps: 366 return None 367 else: 368 return self.steps.pop(0)

369

370 - def startNextStep(self):

371 try: 372 s = self.getNextStep() 373 except StopIteration: 374 s = None 375 if not s: 376 return self.allStepsDone() 377 self.currentStep = s 378 d = defer.maybeDeferred(s.startStep, self.remote) 379 d.addCallback(self._stepDone, s) 380 d.addErrback(self.buildException)

381

382 - def _stepDone(self, results, step):

383 self.currentStep = None 384 if self.finished: 385 return # build was interrupted, don't keep building 386 terminate = self.stepDone(results, step) # interpret/merge results 387 if terminate: 388 self.terminate = True 389 return self.startNextStep()

390

391 - def stepDone(self, result, step):

392 """This method is called when the BuildStep completes. It is passed a 393 status object from the BuildStep and is responsible for merging the 394 Step's results into those of the overall Build.""" 395 396 terminate = False 397 text = None 398 if type(result) == types.TupleType: 399 result, text = result 400 assert type(result) == type(SUCCESS) 401 log.msg(" step '%s' complete: %s" % (step.name, Results[result])) 402 self.results.append(result) 403 if text: 404 self.text.extend(text) 405 if not self.remote: 406 terminate = True 407 408 possible_overall_result = result 409 if result == FAILURE: 410 if not step.flunkOnFailure: 411 possible_overall_result = SUCCESS 412 if step.warnOnFailure: 413 possible_overall_result = WARNINGS 414 if step.flunkOnFailure: 415 possible_overall_result = FAILURE 416 if step.haltOnFailure: 417 terminate = True 418 elif result == WARNINGS: 419 if not step.warnOnWarnings: 420 possible_overall_result = SUCCESS 421 else: 422 possible_overall_result = WARNINGS 423 if step.flunkOnWarnings: 424 possible_overall_result = FAILURE 425 elif result in (EXCEPTION, RETRY): 426 terminate = True 427 428 # if we skipped this step, then don't adjust the build status 429 if result != SKIPPED: 430 self.result = worst_status(self.result, possible_overall_result) 431 432 return terminate

433

434 - def lostRemote(self, remote=None):

435 # the slave went away. There are several possible reasons for this, 436 # and they aren't necessarily fatal. For now, kill the build, but 437 # TODO: see if we can resume the build when it reconnects. 438 log.msg("%s.lostRemote" % self) 439 self.remote = None 440 if self.currentStep: 441 # this should cause the step to finish. 442 log.msg(" stopping currentStep", self.currentStep) 443 self.currentStep.interrupt(Failure(error.ConnectionLost()))

444

445 - def stopBuild(self, reason="<no reason given>"):

446 # the idea here is to let the user cancel a build because, e.g., 447 # they realized they committed a bug and they don't want to waste 448 # the time building something that they know will fail. Another 449 # reason might be to abandon a stuck build. We want to mark the 450 # build as failed quickly rather than waiting for the slave's 451 # timeout to kill it on its own. 452 453 log.msg(" %s: stopping build: %s" % (self, reason)) 454 if self.finished: 455 return 456 # TODO: include 'reason' in this point event 457 self.builder.builder_status.addPointEvent(['interrupt']) 458 self.stopped = True 459 if self.currentStep: 460 self.currentStep.interrupt(reason) 461 462 self.result = EXCEPTION 463 464 if self._acquiringLock: 465 lock, access, d = self._acquiringLock 466 lock.stopWaitingUntilAvailable(self, access, d) 467 d.callback(None)

468

469 - def allStepsDone(self):

470 if self.result == FAILURE: 471 text = ["failed"] 472 elif self.result == WARNINGS: 473 text = ["warnings"] 474 elif self.result == EXCEPTION: 475 text = ["exception"] 476 else: 477 text = ["build", "successful"] 478 text.extend(self.text) 479 return self.buildFinished(text, self.result)

480

481 - def buildException(self, why):

482 log.msg("%s.buildException" % self) 483 log.err(why) 484 self.buildFinished(["build", "exception"], EXCEPTION)

485

486 - def buildFinished(self, text, results):

487 """This method must be called when the last Step has completed. It 488 marks the Build as complete and returns the Builder to the 'idle' 489 state. 490 491 It takes two arguments which describe the overall build status: 492 text, results. 'results' is one of SUCCESS, WARNINGS, or FAILURE. 493 494 If 'results' is SUCCESS or WARNINGS, we will permit any dependant 495 builds to start. If it is 'FAILURE', those builds will be 496 abandoned.""" 497 498 self.finished = True 499 if self.remote: 500 self.remote.dontNotifyOnDisconnect(self.lostRemote) 501 self.results = results 502 503 log.msg(" %s: build finished" % self) 504 self.build_status.setText(text) 505 self.build_status.setResults(results) 506 self.build_status.buildFinished() 507 if self.progress and results == SUCCESS: 508 # XXX: also test a 'timing consistent' flag? 509 log.msg(" setting expectations for next time") 510 self.builder.setExpectations(self.progress) 511 reactor.callLater(0, self.releaseLocks) 512 self.deferred.callback(self) 513 self.deferred = None

514

515 - def releaseLocks(self):

516 if self.locks: 517 log.msg("releaseLocks(%s): %s" % (self, self.locks)) 518 for lock, access in self.locks: 519 if lock.isOwner(self, access): 520 lock.release(self, access) 521 else: 522 # This should only happen if we've been interrupted 523 assert self.stopped

524 525 # IBuildControl 526

527 - def getStatus(self):

528 return self.build_status

529 530 # stopBuild is defined earlier 531 532 components.registerAdapter( 533 lambda build : interfaces.IProperties(build.build_status), 534 Build, interfaces.IProperties) 535

Source Code for Module buildbot.process.build