buildbot.process.build

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 17 import types 18 19 from zope.interface import implements 20 from twisted.python import log, components 21 from twisted.python.failure import Failure 22 from twisted.internet import reactor, defer, error 23 24 from buildbot import interfaces, locks 25 from buildbot.status.results import SUCCESS, WARNINGS, FAILURE, EXCEPTION, \ 26 RETRY, SKIPPED, worst_status 27 from buildbot.status.builder import Results 28 from buildbot.status.progress import BuildProgress 29 from buildbot.process import metrics, properties 30 31

32 -class Build(properties.PropertiesMixin):

33 """I represent a single build by a single slave. Specialized Builders can 34 use subclasses of Build to hold status information unique to those build 35 processes. 36 37 I control B{how} the build proceeds. The actual build is broken up into a 38 series of steps, saved in the .buildSteps[] array as a list of 39 L{buildbot.process.step.BuildStep} objects. Each step is a single remote 40 command, possibly a shell command. 41 42 During the build, I put status information into my C{BuildStatus} 43 gatherer. 44 45 After the build, I go away. 46 47 I can be used by a factory by setting buildClass on 48 L{buildbot.process.factory.BuildFactory} 49 50 @ivar requests: the list of L{BuildRequest}s that triggered me 51 @ivar build_status: the L{buildbot.status.build.BuildStatus} that 52 collects our status 53 """ 54 55 implements(interfaces.IBuildControl) 56 57 workdir = "build" 58 build_status = None 59 reason = "changes" 60 finished = False 61 results = None 62 stopped = False 63 set_runtime_properties = True 64

65 - def __init__(self, requests):

66 self.requests = requests 67 self.locks = [] 68 # build a source stamp 69 self.sources = requests[0].mergeSourceStampsWith(requests[1:]) 70 self.reason = requests[0].mergeReasons(requests[1:]) 71 72 self.progress = None 73 self.currentStep = None 74 self.slaveEnvironment = {} 75 76 self.terminate = False 77 78 self._acquiringLock = None

79

80 - def setBuilder(self, builder):

81 """ 82 Set the given builder as our builder. 83 84 @type builder: L{buildbot.process.builder.Builder} 85 """ 86 self.builder = builder

87

88 - def setLocks(self, locks):

89 self.locks = locks

90

91 - def setSlaveEnvironment(self, env):

92 self.slaveEnvironment = env

93

94 - def getSourceStamp(self, codebase=''):

95 for source in self.sources: 96 if source.codebase == codebase: 97 return source 98 return None

99

100 - def getAllSourceStamps(self):

101 return list(self.sources)

102

103 - def allChanges(self):

104 for s in self.sources: 105 for c in s.changes: 106 yield c

107

108 - def allFiles(self):

109 # return a list of all source files that were changed 110 files = [] 111 for c in self.allChanges(): 112 for f in c.files: 113 files.append(f) 114 return files

115

116 - def __repr__(self):

117 return "<Build %s>" % (self.builder.name,)

118

119 - def blamelist(self):

120 blamelist = [] 121 for c in self.allChanges(): 122 if c.who not in blamelist: 123 blamelist.append(c.who) 124 for source in self.sources: 125 if source.patch_info: #Add patch author to blamelist 126 blamelist.append(source.patch_info[0]) 127 blamelist.sort() 128 return blamelist

129

130 - def changesText(self):

131 changetext = "" 132 for c in self.allChanges(): 133 changetext += "-" * 60 + "\n\n" + c.asText() + "\n" 134 # consider sorting these by number 135 return changetext

136

137 - def setStepFactories(self, step_factories):

138 """Set a list of 'step factories', which are tuples of (class, 139 kwargs), where 'class' is generally a subclass of step.BuildStep . 140 These are used to create the Steps themselves when the Build starts 141 (as opposed to when it is first created). By creating the steps 142 later, their __init__ method will have access to things like 143 build.allFiles() .""" 144 self.stepFactories = list(step_factories)

145 146 useProgress = True 147

148 - def getSlaveCommandVersion(self, command, oldversion=None):

149 return self.slavebuilder.getSlaveCommandVersion(command, oldversion)

150 - def getSlaveName(self):

151 return self.slavebuilder.slave.slavename

152

153 - def setupProperties(self):

154 props = interfaces.IProperties(self) 155 156 # give the properties a reference back to this build 157 props.build = self 158 159 # start with global properties from the configuration 160 buildmaster = self.builder.botmaster.parent 161 props.updateFromProperties(buildmaster.config.properties) 162 163 # from the SourceStamps, which have properties via Change 164 for change in self.allChanges(): 165 props.updateFromProperties(change.properties) 166 167 # and finally, get any properties from requests (this is the path 168 # through which schedulers will send us properties) 169 for rq in self.requests: 170 props.updateFromProperties(rq.properties) 171 172 # now set some properties of our own, corresponding to the 173 # build itself 174 props.setProperty("buildnumber", self.build_status.number, "Build") 175 176 if self.sources and len(self.sources) == 1: 177 # old interface for backwards compatibility 178 source = self.sources[0] 179 props.setProperty("branch", source.branch, "Build") 180 props.setProperty("revision", source.revision, "Build") 181 props.setProperty("repository", source.repository, "Build") 182 props.setProperty("codebase", source.codebase, "Build") 183 props.setProperty("project", source.project, "Build") 184 185 self.builder.setupProperties(props)

186

187 - def setupSlaveBuilder(self, slavebuilder):

188 self.slavebuilder = slavebuilder 189 190 self.path_module = slavebuilder.slave.path_module 191 192 # navigate our way back to the L{buildbot.buildslave.BuildSlave} 193 # object that came from the config, and get its properties 194 buildslave_properties = slavebuilder.slave.properties 195 self.getProperties().updateFromProperties(buildslave_properties) 196 if slavebuilder.slave.slave_basedir: 197 self.setProperty("workdir", 198 self.path_module.join( 199 slavebuilder.slave.slave_basedir, 200 self.builder.config.slavebuilddir), 201 "slave") 202 203 self.slavename = slavebuilder.slave.slavename 204 self.build_status.setSlavename(self.slavename)

205

206 - def startBuild(self, build_status, expectations, slavebuilder):

207 """This method sets up the build, then starts it by invoking the 208 first Step. It returns a Deferred which will fire when the build 209 finishes. This Deferred is guaranteed to never errback.""" 210 211 # we are taking responsibility for watching the connection to the 212 # remote. This responsibility was held by the Builder until our 213 # startBuild was called, and will not return to them until we fire 214 # the Deferred returned by this method. 215 216 log.msg("%s.startBuild" % self) 217 self.build_status = build_status 218 # now that we have a build_status, we can set properties 219 self.setupProperties() 220 self.setupSlaveBuilder(slavebuilder) 221 slavebuilder.slave.updateSlaveStatus(buildStarted=build_status) 222 223 # convert all locks into their real forms 224 lock_list = [] 225 for access in self.locks: 226 if not isinstance(access, locks.LockAccess): 227 # Buildbot 0.7.7 compability: user did not specify access 228 access = access.defaultAccess() 229 lock = self.builder.botmaster.getLockByID(access.lockid) 230 lock_list.append((lock, access)) 231 self.locks = lock_list 232 # then narrow SlaveLocks down to the right slave 233 self.locks = [(l.getLock(self.slavebuilder.slave), la) 234 for l, la in self.locks] 235 self.remote = slavebuilder.remote 236 self.remote.notifyOnDisconnect(self.lostRemote) 237 238 metrics.MetricCountEvent.log('active_builds', 1) 239 240 d = self.deferred = defer.Deferred() 241 def _uncount_build(res): 242 metrics.MetricCountEvent.log('active_builds', -1) 243 return res

244 d.addBoth(_uncount_build) 245 246 def _release_slave(res, slave, bs): 247 self.slavebuilder.buildFinished() 248 slave.updateSlaveStatus(buildFinished=bs) 249 return res

250 d.addCallback(_release_slave, self.slavebuilder.slave, build_status) 251 252 try: 253 self.setupBuild(expectations) # create .steps 254 except: 255 # the build hasn't started yet, so log the exception as a point 256 # event instead of flunking the build. 257 # TODO: associate this failure with the build instead. 258 # this involves doing 259 # self.build_status.buildStarted() from within the exception 260 # handler 261 log.msg("Build.setupBuild failed") 262 log.err(Failure()) 263 self.builder.builder_status.addPointEvent(["setupBuild", 264 "exception"]) 265 self.finished = True 266 self.results = FAILURE 267 self.deferred = None 268 d.callback(self) 269 return d 270 271 self.build_status.buildStarted(self) 272 self.acquireLocks().addCallback(self._startBuild_2) 273 return d 274

275 - def acquireLocks(self, res=None):

276 self._acquiringLock = None 277 if not self.locks: 278 return defer.succeed(None) 279 if self.stopped: 280 return defer.succeed(None) 281 log.msg("acquireLocks(build %s, locks %s)" % (self, self.locks)) 282 for lock, access in self.locks: 283 if not lock.isAvailable(self, access): 284 log.msg("Build %s waiting for lock %s" % (self, lock)) 285 d = lock.waitUntilMaybeAvailable(self, access) 286 d.addCallback(self.acquireLocks) 287 self._acquiringLock = (lock, access, d) 288 return d 289 # all locks are available, claim them all 290 for lock, access in self.locks: 291 lock.claim(self, access) 292 return defer.succeed(None)

293

294 - def _startBuild_2(self, res):

295 self.startNextStep()

296

297 - def setupBuild(self, expectations):

298 # create the actual BuildSteps. If there are any name collisions, we 299 # add a count to the loser until it is unique. 300 self.steps = [] 301 self.stepStatuses = {} 302 stepnames = {} 303 sps = [] 304 305 for factory in self.stepFactories: 306 step = factory.buildStep() 307 step.setBuild(self) 308 step.setBuildSlave(self.slavebuilder.slave) 309 if callable (self.workdir): 310 step.setDefaultWorkdir (self.workdir (self.sources)) 311 else: 312 step.setDefaultWorkdir (self.workdir) 313 name = step.name 314 if stepnames.has_key(name): 315 count = stepnames[name] 316 count += 1 317 stepnames[name] = count 318 name = step.name + "_%d" % count 319 else: 320 stepnames[name] = 0 321 step.name = name 322 self.steps.append(step) 323 324 # tell the BuildStatus about the step. This will create a 325 # BuildStepStatus and bind it to the Step. 326 step_status = self.build_status.addStepWithName(name) 327 step.setStepStatus(step_status) 328 329 sp = None 330 if self.useProgress: 331 # XXX: maybe bail if step.progressMetrics is empty? or skip 332 # progress for that one step (i.e. "it is fast"), or have a 333 # separate "variable" flag that makes us bail on progress 334 # tracking 335 sp = step.setupProgress() 336 if sp: 337 sps.append(sp) 338 339 # Create a buildbot.status.progress.BuildProgress object. This is 340 # called once at startup to figure out how to build the long-term 341 # Expectations object, and again at the start of each build to get a 342 # fresh BuildProgress object to track progress for that individual 343 # build. TODO: revisit at-startup call 344 345 if self.useProgress: 346 self.progress = BuildProgress(sps) 347 if self.progress and expectations: 348 self.progress.setExpectationsFrom(expectations) 349 350 # we are now ready to set up our BuildStatus. 351 # pass all sourcestamps to the buildstatus 352 self.build_status.setSourceStamps(self.sources) 353 self.build_status.setReason(self.reason) 354 self.build_status.setBlamelist(self.blamelist()) 355 self.build_status.setProgress(self.progress) 356 357 # gather owners from build requests 358 owners = [r.properties['owner'] for r in self.requests 359 if r.properties.has_key('owner')] 360 if owners: self.setProperty('owners', owners, self.reason) 361 362 self.results = [] # list of FAILURE, SUCCESS, WARNINGS, SKIPPED 363 self.result = SUCCESS # overall result, may downgrade after each step 364 self.text = [] # list of text string lists (text2)

365

366 - def getNextStep(self):

367 """This method is called to obtain the next BuildStep for this build. 368 When it returns None (or raises a StopIteration exception), the build 369 is complete.""" 370 if not self.steps: 371 return None 372 if not self.remote: 373 return None 374 if self.terminate or self.stopped: 375 # Run any remaining alwaysRun steps, and skip over the others 376 while True: 377 s = self.steps.pop(0) 378 if s.alwaysRun: 379 return s 380 if not self.steps: 381 return None 382 else: 383 return self.steps.pop(0)

384

385 - def startNextStep(self):

386 try: 387 s = self.getNextStep() 388 except StopIteration: 389 s = None 390 if not s: 391 return self.allStepsDone() 392 self.currentStep = s 393 d = defer.maybeDeferred(s.startStep, self.remote) 394 d.addCallback(self._stepDone, s) 395 d.addErrback(self.buildException)

396

397 - def _stepDone(self, results, step):

398 self.currentStep = None 399 if self.finished: 400 return # build was interrupted, don't keep building 401 terminate = self.stepDone(results, step) # interpret/merge results 402 if terminate: 403 self.terminate = True 404 return self.startNextStep()

405

406 - def stepDone(self, result, step):

407 """This method is called when the BuildStep completes. It is passed a 408 status object from the BuildStep and is responsible for merging the 409 Step's results into those of the overall Build.""" 410 411 terminate = False 412 text = None 413 if type(result) == types.TupleType: 414 result, text = result 415 assert type(result) == type(SUCCESS) 416 log.msg(" step '%s' complete: %s" % (step.name, Results[result])) 417 self.results.append(result) 418 if text: 419 self.text.extend(text) 420 if not self.remote: 421 terminate = True 422 423 possible_overall_result = result 424 if result == FAILURE: 425 if not step.flunkOnFailure: 426 possible_overall_result = SUCCESS 427 if step.warnOnFailure: 428 possible_overall_result = WARNINGS 429 if step.flunkOnFailure: 430 possible_overall_result = FAILURE 431 if step.haltOnFailure: 432 terminate = True 433 elif result == WARNINGS: 434 if not step.warnOnWarnings: 435 possible_overall_result = SUCCESS 436 else: 437 possible_overall_result = WARNINGS 438 if step.flunkOnWarnings: 439 possible_overall_result = FAILURE 440 elif result in (EXCEPTION, RETRY): 441 terminate = True 442 443 # if we skipped this step, then don't adjust the build status 444 if result != SKIPPED: 445 self.result = worst_status(self.result, possible_overall_result) 446 447 return terminate

448

449 - def lostRemote(self, remote=None):

450 # the slave went away. There are several possible reasons for this, 451 # and they aren't necessarily fatal. For now, kill the build, but 452 # TODO: see if we can resume the build when it reconnects. 453 log.msg("%s.lostRemote" % self) 454 self.remote = None 455 if self.currentStep: 456 # this should cause the step to finish. 457 log.msg(" stopping currentStep", self.currentStep) 458 self.currentStep.interrupt(Failure(error.ConnectionLost())) 459 else: 460 self.result = RETRY 461 self.text = ["lost", "remote"] 462 self.stopped = True 463 if self._acquiringLock: 464 lock, access, d = self._acquiringLock 465 lock.stopWaitingUntilAvailable(self, access, d) 466 d.callback(None)

467

468 - def stopBuild(self, reason="<no reason given>"):

469 # the idea here is to let the user cancel a build because, e.g., 470 # they realized they committed a bug and they don't want to waste 471 # the time building something that they know will fail. Another 472 # reason might be to abandon a stuck build. We want to mark the 473 # build as failed quickly rather than waiting for the slave's 474 # timeout to kill it on its own. 475 476 log.msg(" %s: stopping build: %s" % (self, reason)) 477 if self.finished: 478 return 479 # TODO: include 'reason' in this point event 480 self.builder.builder_status.addPointEvent(['interrupt']) 481 self.stopped = True 482 if self.currentStep: 483 self.currentStep.interrupt(reason) 484 485 self.result = EXCEPTION 486 487 if self._acquiringLock: 488 lock, access, d = self._acquiringLock 489 lock.stopWaitingUntilAvailable(self, access, d) 490 d.callback(None)

491

492 - def allStepsDone(self):

493 if self.result == FAILURE: 494 text = ["failed"] 495 elif self.result == WARNINGS: 496 text = ["warnings"] 497 elif self.result == EXCEPTION: 498 text = ["exception"] 499 elif self.result == RETRY: 500 text = ["retry"] 501 else: 502 text = ["build", "successful"] 503 text.extend(self.text) 504 return self.buildFinished(text, self.result)

505

506 - def buildException(self, why):

507 log.msg("%s.buildException" % self) 508 log.err(why) 509 # try to finish the build, but since we've already faced an exception, 510 # this may not work well. 511 try: 512 self.buildFinished(["build", "exception"], EXCEPTION) 513 except: 514 log.err(Failure(), 'while finishing a build with an exception')

515

516 - def buildFinished(self, text, results):

517 """This method must be called when the last Step has completed. It 518 marks the Build as complete and returns the Builder to the 'idle' 519 state. 520 521 It takes two arguments which describe the overall build status: 522 text, results. 'results' is one of SUCCESS, WARNINGS, or FAILURE. 523 524 If 'results' is SUCCESS or WARNINGS, we will permit any dependant 525 builds to start. If it is 'FAILURE', those builds will be 526 abandoned.""" 527 528 self.finished = True 529 if self.remote: 530 self.remote.dontNotifyOnDisconnect(self.lostRemote) 531 self.remote = None 532 self.results = results 533 534 log.msg(" %s: build finished" % self) 535 self.build_status.setText(text) 536 self.build_status.setResults(results) 537 self.build_status.buildFinished() 538 if self.progress and results == SUCCESS: 539 # XXX: also test a 'timing consistent' flag? 540 log.msg(" setting expectations for next time") 541 self.builder.setExpectations(self.progress) 542 reactor.callLater(0, self.releaseLocks) 543 self.deferred.callback(self) 544 self.deferred = None

545

546 - def releaseLocks(self):

547 if self.locks: 548 log.msg("releaseLocks(%s): %s" % (self, self.locks)) 549 for lock, access in self.locks: 550 if lock.isOwner(self, access): 551 lock.release(self, access) 552 else: 553 # This should only happen if we've been interrupted 554 assert self.stopped

555 556 # IBuildControl 557

558 - def getStatus(self):

559 return self.build_status

560 561 # stopBuild is defined earlier 562 563 components.registerAdapter( 564 lambda build : interfaces.IProperties(build.build_status), 565 Build, interfaces.IProperties) 566

Source Code for Module buildbot.process.build