buildbot.process.base

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 17 import types 18 19 from zope.interface import implements 20 from twisted.python import log 21 from twisted.python.failure import Failure 22 from twisted.internet import reactor, defer, error 23 24 from buildbot import interfaces, locks 25 from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, EXCEPTION, \ 26 RETRY, SKIPPED, worst_status 27 from buildbot.status.builder import Results 28 from buildbot.status.progress import BuildProgress 29 30

31 -class Build:

32 """I represent a single build by a single slave. Specialized Builders can 33 use subclasses of Build to hold status information unique to those build 34 processes. 35 36 I control B{how} the build proceeds. The actual build is broken up into a 37 series of steps, saved in the .buildSteps[] array as a list of 38 L{buildbot.process.step.BuildStep} objects. Each step is a single remote 39 command, possibly a shell command. 40 41 During the build, I put status information into my C{BuildStatus} 42 gatherer. 43 44 After the build, I go away. 45 46 I can be used by a factory by setting buildClass on 47 L{buildbot.process.factory.BuildFactory} 48 49 @ivar requests: the list of L{BuildRequest}s that triggered me 50 @ivar build_status: the L{buildbot.status.builder.BuildStatus} that 51 collects our status 52 """ 53 54 implements(interfaces.IBuildControl) 55 56 workdir = "build" 57 build_status = None 58 reason = "changes" 59 finished = False 60 results = None 61 stopped = False 62

63 - def __init__(self, requests):

64 self.requests = requests 65 for req in self.requests: 66 req.startCount += 1 67 self.locks = [] 68 # build a source stamp 69 self.source = requests[0].mergeWith(requests[1:]) 70 self.reason = requests[0].mergeReasons(requests[1:]) 71 72 self.progress = None 73 self.currentStep = None 74 self.slaveEnvironment = {} 75 76 self.terminate = False 77 78 self._acquiringLock = None

79

80 - def setBuilder(self, builder):

81 """ 82 Set the given builder as our builder. 83 84 @type builder: L{buildbot.process.builder.Builder} 85 """ 86 self.builder = builder

87

88 - def setLocks(self, locks):

89 self.locks = locks

90

91 - def setSlaveEnvironment(self, env):

92 self.slaveEnvironment = env

93

94 - def getSourceStamp(self):

95 return self.source

96

97 - def setProperty(self, propname, value, source, runtime=True):

98 """Set a property on this build. This may only be called after the 99 build has started, so that it has a BuildStatus object where the 100 properties can live.""" 101 self.build_status.setProperty(propname, value, source, runtime=True)

102

103 - def getProperties(self):

104 return self.build_status.getProperties()

105

106 - def getProperty(self, propname):

107 return self.build_status.getProperty(propname)

108

109 - def allChanges(self):

110 return self.source.changes

111

112 - def allFiles(self):

113 # return a list of all source files that were changed 114 files = [] 115 for c in self.allChanges(): 116 for f in c.files: 117 files.append(f) 118 return files

119

120 - def __repr__(self):

121 return "<Build %s>" % (self.builder.name,)

122

123 - def blamelist(self):

124 blamelist = [] 125 for c in self.allChanges(): 126 if c.who not in blamelist: 127 blamelist.append(c.who) 128 blamelist.sort() 129 return blamelist

130

131 - def changesText(self):

132 changetext = "" 133 for c in self.allChanges(): 134 changetext += "-" * 60 + "\n\n" + c.asText() + "\n" 135 # consider sorting these by number 136 return changetext

137

138 - def setStepFactories(self, step_factories):

139 """Set a list of 'step factories', which are tuples of (class, 140 kwargs), where 'class' is generally a subclass of step.BuildStep . 141 These are used to create the Steps themselves when the Build starts 142 (as opposed to when it is first created). By creating the steps 143 later, their __init__ method will have access to things like 144 build.allFiles() .""" 145 self.stepFactories = list(step_factories)

146 147 148 149 useProgress = True 150

151 - def getSlaveCommandVersion(self, command, oldversion=None):

152 return self.slavebuilder.getSlaveCommandVersion(command, oldversion)

153 - def getSlaveName(self):

154 return self.slavebuilder.slave.slavename

155

156 - def setupProperties(self):

157 props = self.getProperties() 158 159 # start with global properties from the configuration 160 buildmaster = self.builder.botmaster.parent 161 props.updateFromProperties(buildmaster.properties) 162 163 # from the SourceStamp, which has properties via Change 164 for change in self.source.changes: 165 props.updateFromProperties(change.properties) 166 167 # and finally, get any properties from requests (this is the path 168 # through which schedulers will send us properties) 169 for rq in self.requests: 170 props.updateFromProperties(rq.properties) 171 172 # now set some properties of our own, corresponding to the 173 # build itself 174 props.setProperty("buildnumber", self.build_status.number, "Build") 175 props.setProperty("branch", self.source.branch, "Build") 176 props.setProperty("revision", self.source.revision, "Build") 177 props.setProperty("repository", self.source.repository, "Build") 178 props.setProperty("project", self.source.project, "Build") 179 self.builder.setupProperties(props)

180

181 - def setupSlaveBuilder(self, slavebuilder):

182 self.slavebuilder = slavebuilder 183 184 # navigate our way back to the L{buildbot.buildslave.BuildSlave} 185 # object that came from the config, and get its properties 186 buildslave_properties = slavebuilder.slave.properties 187 self.getProperties().updateFromProperties(buildslave_properties) 188 189 self.slavename = slavebuilder.slave.slavename 190 self.build_status.setSlavename(self.slavename)

191

192 - def startBuild(self, build_status, expectations, slavebuilder):

193 """This method sets up the build, then starts it by invoking the 194 first Step. It returns a Deferred which will fire when the build 195 finishes. This Deferred is guaranteed to never errback.""" 196 197 # we are taking responsibility for watching the connection to the 198 # remote. This responsibility was held by the Builder until our 199 # startBuild was called, and will not return to them until we fire 200 # the Deferred returned by this method. 201 202 log.msg("%s.startBuild" % self) 203 self.build_status = build_status 204 # now that we have a build_status, we can set properties 205 self.setupProperties() 206 self.setupSlaveBuilder(slavebuilder) 207 slavebuilder.slave.updateSlaveStatus(buildStarted=build_status) 208 209 # convert all locks into their real forms 210 lock_list = [] 211 for access in self.locks: 212 if not isinstance(access, locks.LockAccess): 213 # Buildbot 0.7.7 compability: user did not specify access 214 access = access.defaultAccess() 215 lock = self.builder.botmaster.getLockByID(access.lockid) 216 lock_list.append((lock, access)) 217 self.locks = lock_list 218 # then narrow SlaveLocks down to the right slave 219 self.locks = [(l.getLock(self.slavebuilder), la) 220 for l, la in self.locks] 221 self.remote = slavebuilder.remote 222 self.remote.notifyOnDisconnect(self.lostRemote) 223 d = self.deferred = defer.Deferred() 224 def _release_slave(res, slave, bs): 225 self.slavebuilder.buildFinished() 226 slave.updateSlaveStatus(buildFinished=bs) 227 return res

228 d.addCallback(_release_slave, self.slavebuilder.slave, build_status) 229 230 try: 231 self.setupBuild(expectations) # create .steps 232 except: 233 # the build hasn't started yet, so log the exception as a point 234 # event instead of flunking the build. 235 # TODO: associate this failure with the build instead. 236 # this involves doing 237 # self.build_status.buildStarted() from within the exception 238 # handler 239 log.msg("Build.setupBuild failed") 240 log.err(Failure()) 241 self.builder.builder_status.addPointEvent(["setupBuild", 242 "exception"]) 243 self.finished = True 244 self.results = FAILURE 245 self.deferred = None 246 d.callback(self) 247 return d 248 249 self.build_status.buildStarted(self) 250 self.acquireLocks().addCallback(self._startBuild_2) 251 return d

252

253 - def acquireLocks(self, res=None):

254 self._acquiringLock = None 255 if not self.locks: 256 return defer.succeed(None) 257 if self.stopped: 258 return defer.succeed(None) 259 log.msg("acquireLocks(build %s, locks %s)" % (self, self.locks)) 260 for lock, access in self.locks: 261 if not lock.isAvailable(access): 262 log.msg("Build %s waiting for lock %s" % (self, lock)) 263 d = lock.waitUntilMaybeAvailable(self, access) 264 d.addCallback(self.acquireLocks) 265 self._acquiringLock = (lock, access, d) 266 return d 267 # all locks are available, claim them all 268 for lock, access in self.locks: 269 lock.claim(self, access) 270 return defer.succeed(None)

271

272 - def _startBuild_2(self, res):

273 self.startNextStep()

274

275 - def setupBuild(self, expectations):

276 # create the actual BuildSteps. If there are any name collisions, we 277 # add a count to the loser until it is unique. 278 self.steps = [] 279 self.stepStatuses = {} 280 stepnames = {} 281 sps = [] 282 283 for factory, args in self.stepFactories: 284 args = args.copy() 285 try: 286 step = factory(**args) 287 except: 288 log.msg("error while creating step, factory=%s, args=%s" 289 % (factory, args)) 290 raise 291 step.setBuild(self) 292 step.setBuildSlave(self.slavebuilder.slave) 293 if callable (self.workdir): 294 step.setDefaultWorkdir (self.workdir (self.source)) 295 else: 296 step.setDefaultWorkdir (self.workdir) 297 name = step.name 298 if stepnames.has_key(name): 299 count = stepnames[name] 300 count += 1 301 stepnames[name] = count 302 name = step.name + "_%d" % count 303 else: 304 stepnames[name] = 0 305 step.name = name 306 self.steps.append(step) 307 308 # tell the BuildStatus about the step. This will create a 309 # BuildStepStatus and bind it to the Step. 310 step_status = self.build_status.addStepWithName(name) 311 step.setStepStatus(step_status) 312 313 sp = None 314 if self.useProgress: 315 # XXX: maybe bail if step.progressMetrics is empty? or skip 316 # progress for that one step (i.e. "it is fast"), or have a 317 # separate "variable" flag that makes us bail on progress 318 # tracking 319 sp = step.setupProgress() 320 if sp: 321 sps.append(sp) 322 323 # Create a buildbot.status.progress.BuildProgress object. This is 324 # called once at startup to figure out how to build the long-term 325 # Expectations object, and again at the start of each build to get a 326 # fresh BuildProgress object to track progress for that individual 327 # build. TODO: revisit at-startup call 328 329 if self.useProgress: 330 self.progress = BuildProgress(sps) 331 if self.progress and expectations: 332 self.progress.setExpectationsFrom(expectations) 333 334 # we are now ready to set up our BuildStatus. 335 self.build_status.setSourceStamp(self.source) 336 self.build_status.setReason(self.reason) 337 self.build_status.setBlamelist(self.blamelist()) 338 self.build_status.setProgress(self.progress) 339 340 # gather owners from build requests 341 owners = [r.properties['owner'] for r in self.requests 342 if r.properties.has_key('owner')] 343 if owners: self.setProperty('owners', owners, self.reason) 344 345 self.results = [] # list of FAILURE, SUCCESS, WARNINGS, SKIPPED 346 self.result = SUCCESS # overall result, may downgrade after each step 347 self.text = [] # list of text string lists (text2)

348

349 - def getNextStep(self):

350 """This method is called to obtain the next BuildStep for this build. 351 When it returns None (or raises a StopIteration exception), the build 352 is complete.""" 353 if self.stopped: 354 return None 355 if not self.steps: 356 return None 357 if not self.remote: 358 return None 359 if self.terminate: 360 while True: 361 s = self.steps.pop(0) 362 if s.alwaysRun: 363 return s 364 if not self.steps: 365 return None 366 else: 367 return self.steps.pop(0)

368

369 - def startNextStep(self):

370 try: 371 s = self.getNextStep() 372 except StopIteration: 373 s = None 374 if not s: 375 return self.allStepsDone() 376 self.currentStep = s 377 d = defer.maybeDeferred(s.startStep, self.remote) 378 d.addCallback(self._stepDone, s) 379 d.addErrback(self.buildException)

380

381 - def _stepDone(self, results, step):

382 self.currentStep = None 383 if self.finished: 384 return # build was interrupted, don't keep building 385 terminate = self.stepDone(results, step) # interpret/merge results 386 if terminate: 387 self.terminate = True 388 return self.startNextStep()

389

390 - def stepDone(self, result, step):

391 """This method is called when the BuildStep completes. It is passed a 392 status object from the BuildStep and is responsible for merging the 393 Step's results into those of the overall Build.""" 394 395 terminate = False 396 text = None 397 if type(result) == types.TupleType: 398 result, text = result 399 assert type(result) == type(SUCCESS) 400 log.msg(" step '%s' complete: %s" % (step.name, Results[result])) 401 self.results.append(result) 402 if text: 403 self.text.extend(text) 404 if not self.remote: 405 terminate = True 406 407 possible_overall_result = result 408 if result == FAILURE: 409 if not step.flunkOnFailure: 410 possible_overall_result = SUCCESS 411 if step.warnOnFailure: 412 possible_overall_result = WARNINGS 413 if step.flunkOnFailure: 414 possible_overall_result = FAILURE 415 if step.haltOnFailure: 416 terminate = True 417 elif result == WARNINGS: 418 if not step.warnOnWarnings: 419 possible_overall_result = SUCCESS 420 else: 421 possible_overall_result = WARNINGS 422 if step.flunkOnWarnings: 423 possible_overall_result = FAILURE 424 elif result in (EXCEPTION, RETRY): 425 terminate = True 426 427 # if we skipped this step, then don't adjust the build status 428 if result != SKIPPED: 429 self.result = worst_status(self.result, possible_overall_result) 430 431 return terminate

432

433 - def lostRemote(self, remote=None):

434 # the slave went away. There are several possible reasons for this, 435 # and they aren't necessarily fatal. For now, kill the build, but 436 # TODO: see if we can resume the build when it reconnects. 437 log.msg("%s.lostRemote" % self) 438 self.remote = None 439 if self.currentStep: 440 # this should cause the step to finish. 441 log.msg(" stopping currentStep", self.currentStep) 442 self.currentStep.interrupt(Failure(error.ConnectionLost()))

443

444 - def stopBuild(self, reason="<no reason given>"):

445 # the idea here is to let the user cancel a build because, e.g., 446 # they realized they committed a bug and they don't want to waste 447 # the time building something that they know will fail. Another 448 # reason might be to abandon a stuck build. We want to mark the 449 # build as failed quickly rather than waiting for the slave's 450 # timeout to kill it on its own. 451 452 log.msg(" %s: stopping build: %s" % (self, reason)) 453 if self.finished: 454 return 455 # TODO: include 'reason' in this point event 456 self.builder.builder_status.addPointEvent(['interrupt']) 457 self.stopped = True 458 if self.currentStep: 459 self.currentStep.interrupt(reason) 460 461 self.result = EXCEPTION 462 463 if self._acquiringLock: 464 lock, access, d = self._acquiringLock 465 lock.stopWaitingUntilAvailable(self, access, d) 466 d.callback(None)

467

468 - def allStepsDone(self):

469 if self.result == FAILURE: 470 text = ["failed"] 471 elif self.result == WARNINGS: 472 text = ["warnings"] 473 elif self.result == EXCEPTION: 474 text = ["exception"] 475 else: 476 text = ["build", "successful"] 477 text.extend(self.text) 478 return self.buildFinished(text, self.result)

479

480 - def buildException(self, why):

481 log.msg("%s.buildException" % self) 482 log.err(why) 483 self.buildFinished(["build", "exception"], EXCEPTION)

484

485 - def buildFinished(self, text, results):

486 """This method must be called when the last Step has completed. It 487 marks the Build as complete and returns the Builder to the 'idle' 488 state. 489 490 It takes two arguments which describe the overall build status: 491 text, results. 'results' is one of SUCCESS, WARNINGS, or FAILURE. 492 493 If 'results' is SUCCESS or WARNINGS, we will permit any dependant 494 builds to start. If it is 'FAILURE', those builds will be 495 abandoned.""" 496 497 self.finished = True 498 if self.remote: 499 self.remote.dontNotifyOnDisconnect(self.lostRemote) 500 self.results = results 501 502 log.msg(" %s: build finished" % self) 503 self.build_status.setText(text) 504 self.build_status.setResults(results) 505 self.build_status.buildFinished() 506 if self.progress and results == SUCCESS: 507 # XXX: also test a 'timing consistent' flag? 508 log.msg(" setting expectations for next time") 509 self.builder.setExpectations(self.progress) 510 reactor.callLater(0, self.releaseLocks) 511 self.deferred.callback(self) 512 self.deferred = None

513

514 - def releaseLocks(self):

515 if self.locks: 516 log.msg("releaseLocks(%s): %s" % (self, self.locks)) 517 for lock, access in self.locks: 518 if lock.isOwner(self, access): 519 lock.release(self, access) 520 else: 521 # This should only happen if we've been interrupted 522 assert self.stopped

523 524 # IBuildControl 525

526 - def getStatus(self):

527 return self.build_status

528 529 # stopBuild is defined earlier 530

Source Code for Module buildbot.process.base