buildbot.status.builder

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 17 import weakref 18 import os, re, itertools 19 from cPickle import load, dump 20 21 from zope.interface import implements 22 from twisted.python import log, runtime 23 from twisted.persisted import styles 24 from buildbot.process import metrics 25 from buildbot import interfaces, util 26 from buildbot.status.event import Event 27 from buildbot.status.build import BuildStatus 28 from buildbot.status.buildrequest import BuildRequestStatus 29 30 # user modules expect these symbols to be present here 31 from buildbot.status.results import SUCCESS, WARNINGS, FAILURE, SKIPPED 32 from buildbot.status.results import EXCEPTION, RETRY, Results, worst_status 33 _hush_pyflakes = [ SUCCESS, WARNINGS, FAILURE, SKIPPED, 34 EXCEPTION, RETRY, Results, worst_status ] 35

36 -class BuilderStatus(styles.Versioned):

37 """I handle status information for a single process.build.Builder object. 38 That object sends status changes to me (frequently as Events), and I 39 provide them on demand to the various status recipients, like the HTML 40 waterfall display and the live status clients. It also sends build 41 summaries to me, which I log and provide to status clients who aren't 42 interested in seeing details of the individual build steps. 43 44 I am responsible for maintaining the list of historic Events and Builds, 45 pruning old ones, and loading them from / saving them to disk. 46 47 I live in the buildbot.process.build.Builder object, in the 48 .builder_status attribute. 49 50 @type category: string 51 @ivar category: user-defined category this builder belongs to; can be 52 used to filter on in status clients 53 """ 54 55 implements(interfaces.IBuilderStatus, interfaces.IEventSource) 56 57 persistenceVersion = 1 58 persistenceForgets = ( 'wasUpgraded', ) 59 60 # these limit the amount of memory we consume, as well as the size of the 61 # main Builder pickle. The Build and LogFile pickles on disk must be 62 # handled separately. 63 buildCacheSize = 15 64 eventHorizon = 50 # forget events beyond this 65 66 # these limit on-disk storage 67 logHorizon = 40 # forget logs in steps in builds beyond this 68 buildHorizon = 100 # forget builds beyond this 69 70 category = None 71 currentBigState = "offline" # or idle/waiting/interlocked/building 72 basedir = None # filled in by our parent 73

74 - def __init__(self, buildername, category=None):

75 self.name = buildername 76 self.category = category 77 78 self.slavenames = [] 79 self.events = [] 80 # these three hold Events, and are used to retrieve the current 81 # state of the boxes. 82 self.lastBuildStatus = None 83 #self.currentBig = None 84 #self.currentSmall = None 85 self.currentBuilds = [] 86 self.nextBuild = None 87 self.watchers = [] 88 self.buildCache = weakref.WeakValueDictionary() 89 self.buildCache_LRU = [] 90 self.logCompressionLimit = False # default to no compression for tests 91 self.logCompressionMethod = "bz2" 92 self.logMaxSize = None # No default limit 93 self.logMaxTailSize = None # No tail buffering

94 95 # persistence 96

97 - def __getstate__(self):

98 # when saving, don't record transient stuff like what builds are 99 # currently running, because they won't be there when we start back 100 # up. Nor do we save self.watchers, nor anything that gets set by our 101 # parent like .basedir and .status 102 d = styles.Versioned.__getstate__(self) 103 d['watchers'] = [] 104 del d['buildCache'] 105 del d['buildCache_LRU'] 106 for b in self.currentBuilds: 107 b.saveYourself() 108 # TODO: push a 'hey, build was interrupted' event 109 del d['currentBuilds'] 110 d.pop('pendingBuilds', None) 111 del d['currentBigState'] 112 del d['basedir'] 113 del d['status'] 114 del d['nextBuildNumber'] 115 return d

116

117 - def __setstate__(self, d):

118 # when loading, re-initialize the transient stuff. Remember that 119 # upgradeToVersion1 and such will be called after this finishes. 120 styles.Versioned.__setstate__(self, d) 121 self.buildCache = weakref.WeakValueDictionary() 122 self.buildCache_LRU = [] 123 self.currentBuilds = [] 124 self.watchers = [] 125 self.slavenames = []

126 # self.basedir must be filled in by our parent 127 # self.status must be filled in by our parent 128

129 - def reconfigFromBuildmaster(self, buildmaster):

130 # Note that we do not hang onto the buildmaster, since this object 131 # gets pickled and unpickled. 132 if buildmaster.buildCacheSize is not None: 133 self.buildCacheSize = buildmaster.buildCacheSize

134

135 - def upgradeToVersion1(self):

136 if hasattr(self, 'slavename'): 137 self.slavenames = [self.slavename] 138 del self.slavename 139 if hasattr(self, 'nextBuildNumber'): 140 del self.nextBuildNumber # determineNextBuildNumber chooses this 141 self.wasUpgraded = True

142

143 - def determineNextBuildNumber(self):

144 """Scan our directory of saved BuildStatus instances to determine 145 what our self.nextBuildNumber should be. Set it one larger than the 146 highest-numbered build we discover. This is called by the top-level 147 Status object shortly after we are created or loaded from disk. 148 """ 149 existing_builds = [int(f) 150 for f in os.listdir(self.basedir) 151 if re.match("^\d+$", f)] 152 if existing_builds: 153 self.nextBuildNumber = max(existing_builds) + 1 154 else: 155 self.nextBuildNumber = 0

156

157 - def setLogCompressionLimit(self, lowerLimit):

158 self.logCompressionLimit = lowerLimit

159

160 - def setLogCompressionMethod(self, method):

161 assert method in ("bz2", "gz") 162 self.logCompressionMethod = method

163

164 - def setLogMaxSize(self, upperLimit):

165 self.logMaxSize = upperLimit

166

167 - def setLogMaxTailSize(self, tailSize):

168 self.logMaxTailSize = tailSize

169

170 - def saveYourself(self):

171 for b in self.currentBuilds: 172 if not b.isFinished: 173 # interrupted build, need to save it anyway. 174 # BuildStatus.saveYourself will mark it as interrupted. 175 b.saveYourself() 176 filename = os.path.join(self.basedir, "builder") 177 tmpfilename = filename + ".tmp" 178 try: 179 dump(self, open(tmpfilename, "wb"), -1) 180 if runtime.platformType == 'win32': 181 # windows cannot rename a file on top of an existing one 182 if os.path.exists(filename): 183 os.unlink(filename) 184 os.rename(tmpfilename, filename) 185 except: 186 log.msg("unable to save builder %s" % self.name) 187 log.err()

188 189 190 # build cache management 191

192 - def makeBuildFilename(self, number):

193 return os.path.join(self.basedir, "%d" % number)

194

195 - def touchBuildCache(self, build):

196 self.buildCache[build.number] = build 197 if build in self.buildCache_LRU: 198 self.buildCache_LRU.remove(build) 199 self.buildCache_LRU = self.buildCache_LRU[-(self.buildCacheSize-1):] + [ build ] 200 return build

201

202 - def getBuildByNumber(self, number):

203 # first look in currentBuilds 204 for b in self.currentBuilds: 205 if b.number == number: 206 return self.touchBuildCache(b) 207 208 # then in the buildCache 209 if number in self.buildCache: 210 metrics.MetricCountEvent.log("buildCache.hits", 1) 211 return self.touchBuildCache(self.buildCache[number]) 212 metrics.MetricCountEvent.log("buildCache.misses", 1) 213 214 # then fall back to loading it from disk 215 filename = self.makeBuildFilename(number) 216 try: 217 log.msg("Loading builder %s's build %d from on-disk pickle" 218 % (self.name, number)) 219 build = load(open(filename, "rb")) 220 build.builder = self 221 222 # (bug #1068) if we need to upgrade, we probably need to rewrite 223 # this pickle, too. We determine this by looking at the list of 224 # Versioned objects that have been unpickled, and (after doUpgrade) 225 # checking to see if any of them set wasUpgraded. The Versioneds' 226 # upgradeToVersionNN methods all set this. 227 versioneds = styles.versionedsToUpgrade 228 styles.doUpgrade() 229 if True in [ hasattr(o, 'wasUpgraded') for o in versioneds.values() ]: 230 log.msg("re-writing upgraded build pickle") 231 build.saveYourself() 232 233 # handle LogFiles from after 0.5.0 and before 0.6.5 234 build.upgradeLogfiles() 235 # check that logfiles exist 236 build.checkLogfiles() 237 return self.touchBuildCache(build) 238 except IOError: 239 raise IndexError("no such build %d" % number) 240 except EOFError: 241 raise IndexError("corrupted build pickle %d" % number)

242

243 - def prune(self, events_only=False):

244 # begin by pruning our own events 245 self.events = self.events[-self.eventHorizon:] 246 247 if events_only: 248 return 249 250 # get the horizons straight 251 if self.buildHorizon is not None: 252 earliest_build = self.nextBuildNumber - self.buildHorizon 253 else: 254 earliest_build = 0 255 256 if self.logHorizon is not None: 257 earliest_log = self.nextBuildNumber - self.logHorizon 258 else: 259 earliest_log = 0 260 261 if earliest_log < earliest_build: 262 earliest_log = earliest_build 263 264 if earliest_build == 0: 265 return 266 267 # skim the directory and delete anything that shouldn't be there anymore 268 build_re = re.compile(r"^([0-9]+)$") 269 build_log_re = re.compile(r"^([0-9]+)-.*$") 270 # if the directory doesn't exist, bail out here 271 if not os.path.exists(self.basedir): 272 return 273 274 for filename in os.listdir(self.basedir): 275 num = None 276 mo = build_re.match(filename) 277 is_logfile = False 278 if mo: 279 num = int(mo.group(1)) 280 else: 281 mo = build_log_re.match(filename) 282 if mo: 283 num = int(mo.group(1)) 284 is_logfile = True 285 286 if num is None: continue 287 if num in self.buildCache: continue 288 289 if (is_logfile and num < earliest_log) or num < earliest_build: 290 pathname = os.path.join(self.basedir, filename) 291 log.msg("pruning '%s'" % pathname) 292 try: os.unlink(pathname) 293 except OSError: pass

294 295 # IBuilderStatus methods

296 - def getName(self):

297 return self.name

298

299 - def getState(self):

300 return (self.currentBigState, self.currentBuilds)

301

302 - def getSlaves(self):

303 return [self.status.getSlave(name) for name in self.slavenames]

304

305 - def getPendingBuildRequestStatuses(self):

306 db = self.status.master.db 307 d = db.buildrequests.getBuildRequests(claimed=False, 308 buildername=self.name) 309 def make_statuses(brdicts): 310 return [BuildRequestStatus(self.name, brdict['brid'], 311 self.status) 312 for brdict in brdicts]

313 d.addCallback(make_statuses) 314 return d

315

316 - def getCurrentBuilds(self):

317 return self.currentBuilds

318

319 - def getLastFinishedBuild(self):

320 b = self.getBuild(-1) 321 if not (b and b.isFinished()): 322 b = self.getBuild(-2) 323 return b

324

325 - def getCategory(self):

326 return self.category

327

328 - def getBuild(self, number):

329 if number < 0: 330 number = self.nextBuildNumber + number 331 if number < 0 or number >= self.nextBuildNumber: 332 return None 333 334 try: 335 return self.getBuildByNumber(number) 336 except IndexError: 337 return None

338

339 - def getEvent(self, number):

340 try: 341 return self.events[number] 342 except IndexError: 343 return None

344

345 - def generateFinishedBuilds(self, branches=[], 346 num_builds=None, 347 max_buildnum=None, 348 finished_before=None, 349 max_search=200):

350 got = 0 351 for Nb in itertools.count(1): 352 if Nb > self.nextBuildNumber: 353 break 354 if Nb > max_search: 355 break 356 build = self.getBuild(-Nb) 357 if build is None: 358 continue 359 if max_buildnum is not None: 360 if build.getNumber() > max_buildnum: 361 continue 362 if not build.isFinished(): 363 continue 364 if finished_before is not None: 365 start, end = build.getTimes() 366 if end >= finished_before: 367 continue 368 if branches: 369 if build.getSourceStamp().branch not in branches: 370 continue 371 got += 1 372 yield build 373 if num_builds is not None: 374 if got >= num_builds: 375 return

376

377 - def eventGenerator(self, branches=[], categories=[], committers=[], minTime=0):

378 """This function creates a generator which will provide all of this 379 Builder's status events, starting with the most recent and 380 progressing backwards in time. """ 381 382 # remember the oldest-to-earliest flow here. "next" means earlier. 383 384 # TODO: interleave build steps and self.events by timestamp. 385 # TODO: um, I think we're already doing that. 386 387 # TODO: there's probably something clever we could do here to 388 # interleave two event streams (one from self.getBuild and the other 389 # from self.getEvent), which would be simpler than this control flow 390 391 eventIndex = -1 392 e = self.getEvent(eventIndex) 393 for Nb in range(1, self.nextBuildNumber+1): 394 b = self.getBuild(-Nb) 395 if not b: 396 # HACK: If this is the first build we are looking at, it is 397 # possible it's in progress but locked before it has written a 398 # pickle; in this case keep looking. 399 if Nb == 1: 400 continue 401 break 402 if b.getTimes()[0] < minTime: 403 break 404 if branches and not b.getSourceStamp().branch in branches: 405 continue 406 if categories and not b.getBuilder().getCategory() in categories: 407 continue 408 if committers and not [True for c in b.getChanges() if c.who in committers]: 409 continue 410 steps = b.getSteps() 411 for Ns in range(1, len(steps)+1): 412 if steps[-Ns].started: 413 step_start = steps[-Ns].getTimes()[0] 414 while e is not None and e.getTimes()[0] > step_start: 415 yield e 416 eventIndex -= 1 417 e = self.getEvent(eventIndex) 418 yield steps[-Ns] 419 yield b 420 while e is not None: 421 yield e 422 eventIndex -= 1 423 e = self.getEvent(eventIndex) 424 if e and e.getTimes()[0] < minTime: 425 break

426 436

437 - def unsubscribe(self, receiver):

438 self.watchers.remove(receiver) 439 self.status._builder_unsubscribe(self.name, receiver)

440 441 ## Builder interface (methods called by the Builder which feeds us) 442

443 - def setSlavenames(self, names):

444 self.slavenames = names

445

446 - def addEvent(self, text=[]):

447 # this adds a duration event. When it is done, the user should call 448 # e.finish(). They can also mangle it by modifying .text 449 e = Event() 450 e.started = util.now() 451 e.text = text 452 self.events.append(e) 453 self.prune(events_only=True) 454 return e # they are free to mangle it further

455

456 - def addPointEvent(self, text=[]):

457 # this adds a point event, one which occurs as a single atomic 458 # instant of time. 459 e = Event() 460 e.started = util.now() 461 e.finished = 0 462 e.text = text 463 self.events.append(e) 464 self.prune(events_only=True) 465 return e # for consistency, but they really shouldn't touch it

466

467 - def setBigState(self, state):

468 needToUpdate = state != self.currentBigState 469 self.currentBigState = state 470 if needToUpdate: 471 self.publishState()

472

473 - def publishState(self, target=None):

474 state = self.currentBigState 475 476 if target is not None: 477 # unicast 478 target.builderChangedState(self.name, state) 479 return 480 for w in self.watchers: 481 try: 482 w.builderChangedState(self.name, state) 483 except: 484 log.msg("Exception caught publishing state to %r" % w) 485 log.err()

486

487 - def newBuild(self):

488 """The Builder has decided to start a build, but the Build object is 489 not yet ready to report status (it has not finished creating the 490 Steps). Create a BuildStatus object that it can use.""" 491 number = self.nextBuildNumber 492 self.nextBuildNumber += 1 493 # TODO: self.saveYourself(), to make sure we don't forget about the 494 # build number we've just allocated. This is not quite as important 495 # as it was before we switch to determineNextBuildNumber, but I think 496 # it may still be useful to have the new build save itself. 497 s = BuildStatus(self, number) 498 s.waitUntilFinished().addCallback(self._buildFinished) 499 return s

500 501 # buildStarted is called by our child BuildStatus instances

502 - def buildStarted(self, s):

503 """Now the BuildStatus object is ready to go (it knows all of its 504 Steps, its ETA, etc), so it is safe to notify our watchers.""" 505 506 assert s.builder is self # paranoia 507 assert s not in self.currentBuilds 508 self.currentBuilds.append(s) 509 self.touchBuildCache(s) 510 511 # now that the BuildStatus is prepared to answer queries, we can 512 # announce the new build to all our watchers 513 514 for w in self.watchers: # TODO: maybe do this later? callLater(0)? 515 try: 516 receiver = w.buildStarted(self.getName(), s) 517 if receiver: 518 if type(receiver) == type(()): 519 s.subscribe(receiver[0], receiver[1]) 520 else: 521 s.subscribe(receiver) 522 d = s.waitUntilFinished() 523 d.addCallback(lambda s: s.unsubscribe(receiver)) 524 except: 525 log.msg("Exception caught notifying %r of buildStarted event" % w) 526 log.err()

527

528 - def _buildFinished(self, s):

529 assert s in self.currentBuilds 530 s.saveYourself() 531 self.currentBuilds.remove(s) 532 533 name = self.getName() 534 results = s.getResults() 535 for w in self.watchers: 536 try: 537 w.buildFinished(name, s, results) 538 except: 539 log.msg("Exception caught notifying %r of buildFinished event" % w) 540 log.err() 541 542 self.prune() # conserve disk

543 544 545 # waterfall display (history) 546 547 # I want some kind of build event that holds everything about the build: 548 # why, what changes went into it, the results of the build, itemized 549 # test results, etc. But, I do kind of need something to be inserted in 550 # the event log first, because intermixing step events and the larger 551 # build event is fraught with peril. Maybe an Event-like-thing that 552 # doesn't have a file in it but does have links. Hmm, that's exactly 553 # what it does now. The only difference would be that this event isn't 554 # pushed to the clients. 555 556 # publish to clients 557 ## HTML display interface 558

559 - def getEventNumbered(self, num):

560 # deal with dropped events, pruned events 561 first = self.events[0].number 562 if first + len(self.events)-1 != self.events[-1].number: 563 log.msg(self, 564 "lost an event somewhere: [0] is %d, [%d] is %d" % \ 565 (self.events[0].number, 566 len(self.events) - 1, 567 self.events[-1].number)) 568 for e in self.events: 569 log.msg("e[%d]: " % e.number, e) 570 return None 571 offset = num - first 572 log.msg(self, "offset", offset) 573 try: 574 return self.events[offset] 575 except IndexError: 576 return None

577 578 ## Persistence of Status

579 - def loadYourOldEvents(self):

580 if hasattr(self, "allEvents"): 581 # first time, nothing to get from file. Note that this is only if 582 # the Application gets .run() . If it gets .save()'ed, then the 583 # .allEvents attribute goes away in the initial __getstate__ and 584 # we try to load a non-existent file. 585 return 586 self.allEvents = self.loadFile("events", []) 587 if self.allEvents: 588 self.nextEventNumber = self.allEvents[-1].number + 1 589 else: 590 self.nextEventNumber = 0

591 - def saveYourOldEvents(self):

592 self.saveFile("events", self.allEvents)

593 594 ## clients 595

596 - def addClient(self, client):

597 if client not in self.subscribers: 598 self.subscribers.append(client) 599 self.sendCurrentActivityBig(client) 600 client.newEvent(self.currentSmall)

601 - def removeClient(self, client):

602 if client in self.subscribers: 603 self.subscribers.remove(client)

604

605 - def asDict(self):

606 result = {} 607 # Constant 608 # TODO(maruel): Fix me. We don't want to leak the full path. 609 result['basedir'] = os.path.basename(self.basedir) 610 result['category'] = self.category 611 result['slaves'] = self.slavenames 612 #result['url'] = self.parent.getURLForThing(self) 613 # TODO(maruel): Add cache settings? Do we care? 614 615 # Transient 616 # Collect build numbers. 617 # Important: Only grab the *cached* builds numbers to reduce I/O. 618 current_builds = [b.getNumber() for b in self.currentBuilds] 619 cached_builds = list(set(self.buildCache.keys() + current_builds)) 620 cached_builds.sort() 621 result['cachedBuilds'] = cached_builds 622 result['currentBuilds'] = current_builds 623 result['state'] = self.getState()[0] 624 # lies, but we don't have synchronous access to this info; use 625 # asDict_async instead 626 result['pendingBuilds'] = 0 627 return result

628

629 - def asDict_async(self):

630 """Just like L{asDict}, but with a nonzero pendingBuilds.""" 631 result = self.asDict() 632 d = self.getPendingBuildRequestStatuses() 633 def combine(statuses): 634 result['pendingBuilds'] = len(statuses) 635 return result

636 d.addCallback(combine) 637 return d 638

639 - def getMetrics(self):

640 return self.botmaster.parent.metrics

641 642 # vim: set ts=4 sts=4 sw=4 et: 643

Source Code for Module buildbot.status.builder