buildbot.status.builder

1 # -*- test-case-name: buildbot.test.test_status -*- 2 3 from zope.interface import implements 4 from twisted.python import log 5 from twisted.persisted import styles 6 from twisted.internet import reactor, defer, threads 7 from twisted.protocols import basic 8 from buildbot.process.properties import Properties 9 10 import weakref 11 import os, shutil, sys, re, urllib, itertools 12 import gc 13 from cPickle import load, dump 14 from cStringIO import StringIO 15 16 try: # bz2 is not available on py23 17 from bz2 import BZ2File 18 except ImportError: 19 BZ2File = None 20 21 try: 22 from gzip import GzipFile 23 except ImportError: 24 GzipFile = None 25 26 # sibling imports 27 from buildbot import interfaces, util, sourcestamp 28 29 SUCCESS, WARNINGS, FAILURE, SKIPPED, EXCEPTION = range(5) 30 Results = ["success", "warnings", "failure", "skipped", "exception"] 31 32 33 # build processes call the following methods: 34 # 35 # setDefaults 36 # 37 # currentlyBuilding 38 # currentlyIdle 39 # currentlyInterlocked 40 # currentlyOffline 41 # currentlyWaiting 42 # 43 # setCurrentActivity 44 # updateCurrentActivity 45 # addFileToCurrentActivity 46 # finishCurrentActivity 47 # 48 # startBuild 49 # finishBuild 50 51 STDOUT = interfaces.LOG_CHANNEL_STDOUT 52 STDERR = interfaces.LOG_CHANNEL_STDERR 53 HEADER = interfaces.LOG_CHANNEL_HEADER 54 ChunkTypes = ["stdout", "stderr", "header"] 55

56 -class LogFileScanner(basic.NetstringReceiver):

57 - def __init__(self, chunk_cb, channels=[]):

58 self.chunk_cb = chunk_cb 59 self.channels = channels

60

61 - def stringReceived(self, line):

62 channel = int(line[0]) 63 if not self.channels or (channel in self.channels): 64 self.chunk_cb((channel, line[1:]))

65

66 -class LogFileProducer:

67 """What's the plan? 68 69 the LogFile has just one FD, used for both reading and writing. 70 Each time you add an entry, fd.seek to the end and then write. 71 72 Each reader (i.e. Producer) keeps track of their own offset. The reader 73 starts by seeking to the start of the logfile, and reading forwards. 74 Between each hunk of file they yield chunks, so they must remember their 75 offset before yielding and re-seek back to that offset before reading 76 more data. When their read() returns EOF, they're finished with the first 77 phase of the reading (everything that's already been written to disk). 78 79 After EOF, the remaining data is entirely in the current entries list. 80 These entries are all of the same channel, so we can do one "".join and 81 obtain a single chunk to be sent to the listener. But since that involves 82 a yield, and more data might arrive after we give up control, we have to 83 subscribe them before yielding. We can't subscribe them any earlier, 84 otherwise they'd get data out of order. 85 86 We're using a generator in the first place so that the listener can 87 throttle us, which means they're pulling. But the subscription means 88 we're pushing. Really we're a Producer. In the first phase we can be 89 either a PullProducer or a PushProducer. In the second phase we're only a 90 PushProducer. 91 92 So the client gives a LogFileConsumer to File.subscribeConsumer . This 93 Consumer must have registerProducer(), unregisterProducer(), and 94 writeChunk(), and is just like a regular twisted.interfaces.IConsumer, 95 except that writeChunk() takes chunks (tuples of (channel,text)) instead 96 of the normal write() which takes just text. The LogFileConsumer is 97 allowed to call stopProducing, pauseProducing, and resumeProducing on the 98 producer instance it is given. """ 99 100 paused = False 101 subscribed = False 102 BUFFERSIZE = 2048 103

104 - def __init__(self, logfile, consumer):

105 self.logfile = logfile 106 self.consumer = consumer 107 self.chunkGenerator = self.getChunks() 108 consumer.registerProducer(self, True)

109

110 - def getChunks(self):

111 f = self.logfile.getFile() 112 offset = 0 113 chunks = [] 114 p = LogFileScanner(chunks.append) 115 f.seek(offset) 116 data = f.read(self.BUFFERSIZE) 117 offset = f.tell() 118 while data: 119 p.dataReceived(data) 120 while chunks: 121 c = chunks.pop(0) 122 yield c 123 f.seek(offset) 124 data = f.read(self.BUFFERSIZE) 125 offset = f.tell() 126 del f 127 128 # now subscribe them to receive new entries 129 self.subscribed = True 130 self.logfile.watchers.append(self) 131 d = self.logfile.waitUntilFinished() 132 133 # then give them the not-yet-merged data 134 if self.logfile.runEntries: 135 channel = self.logfile.runEntries[0][0] 136 text = "".join([c[1] for c in self.logfile.runEntries]) 137 yield (channel, text) 138 139 # now we've caught up to the present. Anything further will come from 140 # the logfile subscription. We add the callback *after* yielding the 141 # data from runEntries, because the logfile might have finished 142 # during the yield. 143 d.addCallback(self.logfileFinished)

144

145 - def stopProducing(self):

146 # TODO: should we still call consumer.finish? probably not. 147 self.paused = True 148 self.consumer = None 149 self.done()

150

151 - def done(self):

152 if self.chunkGenerator: 153 self.chunkGenerator = None # stop making chunks 154 if self.subscribed: 155 self.logfile.watchers.remove(self) 156 self.subscribed = False

157

158 - def pauseProducing(self):

159 self.paused = True

160

161 - def resumeProducing(self):

162 # Twisted-1.3.0 has a bug which causes hangs when resumeProducing 163 # calls transport.write (there is a recursive loop, fixed in 2.0 in 164 # t.i.abstract.FileDescriptor.doWrite by setting the producerPaused 165 # flag *before* calling resumeProducing). To work around this, we 166 # just put off the real resumeProducing for a moment. This probably 167 # has a performance hit, but I'm going to assume that the log files 168 # are not retrieved frequently enough for it to be an issue. 169 170 reactor.callLater(0, self._resumeProducing)

171

172 - def _resumeProducing(self):

173 self.paused = False 174 if not self.chunkGenerator: 175 return 176 try: 177 while not self.paused: 178 chunk = self.chunkGenerator.next() 179 self.consumer.writeChunk(chunk) 180 # we exit this when the consumer says to stop, or we run out 181 # of chunks 182 except StopIteration: 183 # if the generator finished, it will have done releaseFile 184 self.chunkGenerator = None

185 # now everything goes through the subscription, and they don't get to 186 # pause anymore 187

188 - def logChunk(self, build, step, logfile, channel, chunk):

189 if self.consumer: 190 self.consumer.writeChunk((channel, chunk))

191

192 - def logfileFinished(self, logfile):

193 self.done() 194 if self.consumer: 195 self.consumer.unregisterProducer() 196 self.consumer.finish() 197 self.consumer = None

198

199 -def _tryremove(filename, timeout, retries):

200 """Try to remove a file, and if failed, try again in timeout. 201 Increases the timeout by a factor of 4, and only keeps trying for 202 another retries-amount of times. 203 204 """ 205 try: 206 os.unlink(filename) 207 except OSError: 208 if retries > 0: 209 reactor.callLater(timeout, _tryremove, filename, timeout * 4, 210 retries - 1) 211 else: 212 log.msg("giving up on removing %s after over %d seconds" % 213 (filename, timeout))

214

215 -class LogFile:

216 """A LogFile keeps all of its contents on disk, in a non-pickle format to 217 which new entries can easily be appended. The file on disk has a name 218 like 12-log-compile-output, under the Builder's directory. The actual 219 filename is generated (before the LogFile is created) by 220 L{BuildStatus.generateLogfileName}. 221 222 Old LogFile pickles (which kept their contents in .entries) must be 223 upgraded. The L{BuilderStatus} is responsible for doing this, when it 224 loads the L{BuildStatus} into memory. The Build pickle is not modified, 225 so users who go from 0.6.5 back to 0.6.4 don't have to lose their 226 logs.""" 227 228 implements(interfaces.IStatusLog, interfaces.ILogFile) 229 230 finished = False 231 length = 0 232 nonHeaderLength = 0 233 tailLength = 0 234 chunkSize = 10*1000 235 runLength = 0 236 # No max size by default 237 logMaxSize = None 238 # Don't keep a tail buffer by default 239 logMaxTailSize = None 240 maxLengthExceeded = False 241 runEntries = [] # provided so old pickled builds will getChunks() ok 242 entries = None 243 BUFFERSIZE = 2048 244 filename = None # relative to the Builder's basedir 245 openfile = None 246 compressMethod = "bz2" 247

248 - def __init__(self, parent, name, logfilename):

249 """ 250 @type parent: L{BuildStepStatus} 251 @param parent: the Step that this log is a part of 252 @type name: string 253 @param name: the name of this log, typically 'output' 254 @type logfilename: string 255 @param logfilename: the Builder-relative pathname for the saved entries 256 """ 257 self.step = parent 258 self.name = name 259 self.filename = logfilename 260 fn = self.getFilename() 261 if os.path.exists(fn): 262 # the buildmaster was probably stopped abruptly, before the 263 # BuilderStatus could be saved, so BuilderStatus.nextBuildNumber 264 # is out of date, and we're overlapping with earlier builds now. 265 # Warn about it, but then overwrite the old pickle file 266 log.msg("Warning: Overwriting old serialized Build at %s" % fn) 267 dirname = os.path.dirname(fn) 268 if not os.path.exists(dirname): 269 os.makedirs(dirname) 270 self.openfile = open(fn, "w+") 271 self.runEntries = [] 272 self.watchers = [] 273 self.finishedWatchers = [] 274 self.tailBuffer = []

275

276 - def getFilename(self):

277 return os.path.join(self.step.build.builder.basedir, self.filename)

278

279 - def hasContents(self):

280 return os.path.exists(self.getFilename() + '.bz2') or \ 281 os.path.exists(self.getFilename() + '.gz') or \ 282 os.path.exists(self.getFilename())

283

284 - def getName(self):

285 return self.name

286

287 - def getStep(self):

288 return self.step

289

290 - def isFinished(self):

291 return self.finished

292 - def waitUntilFinished(self):

293 if self.finished: 294 d = defer.succeed(self) 295 else: 296 d = defer.Deferred() 297 self.finishedWatchers.append(d) 298 return d

299

300 - def getFile(self):

301 if self.openfile: 302 # this is the filehandle we're using to write to the log, so 303 # don't close it! 304 return self.openfile 305 # otherwise they get their own read-only handle 306 # try a compressed log first 307 if BZ2File is not None: 308 try: 309 return BZ2File(self.getFilename() + ".bz2", "r") 310 except IOError: 311 pass 312 if GzipFile is not None: 313 try: 314 return GzipFile(self.getFilename() + ".gz", "r") 315 except IOError: 316 pass 317 return open(self.getFilename(), "r")

318

319 - def getText(self):

320 # this produces one ginormous string 321 return "".join(self.getChunks([STDOUT, STDERR], onlyText=True))

322

323 - def getTextWithHeaders(self):

324 return "".join(self.getChunks(onlyText=True))

325

326 - def getChunks(self, channels=[], onlyText=False):

327 # generate chunks for everything that was logged at the time we were 328 # first called, so remember how long the file was when we started. 329 # Don't read beyond that point. The current contents of 330 # self.runEntries will follow. 331 332 # this returns an iterator, which means arbitrary things could happen 333 # while we're yielding. This will faithfully deliver the log as it 334 # existed when it was started, and not return anything after that 335 # point. To use this in subscribe(catchup=True) without missing any 336 # data, you must insure that nothing will be added to the log during 337 # yield() calls. 338 339 f = self.getFile() 340 if not self.finished: 341 offset = 0 342 f.seek(0, 2) 343 remaining = f.tell() 344 else: 345 offset = 0 346 remaining = None 347 348 leftover = None 349 if self.runEntries and (not channels or 350 (self.runEntries[0][0] in channels)): 351 leftover = (self.runEntries[0][0], 352 "".join([c[1] for c in self.runEntries])) 353 354 # freeze the state of the LogFile by passing a lot of parameters into 355 # a generator 356 return self._generateChunks(f, offset, remaining, leftover, 357 channels, onlyText)

358

359 - def _generateChunks(self, f, offset, remaining, leftover, 360 channels, onlyText):

361 chunks = [] 362 p = LogFileScanner(chunks.append, channels) 363 f.seek(offset) 364 if remaining is not None: 365 data = f.read(min(remaining, self.BUFFERSIZE)) 366 remaining -= len(data) 367 else: 368 data = f.read(self.BUFFERSIZE) 369 370 offset = f.tell() 371 while data: 372 p.dataReceived(data) 373 while chunks: 374 channel, text = chunks.pop(0) 375 if onlyText: 376 yield text 377 else: 378 yield (channel, text) 379 f.seek(offset) 380 if remaining is not None: 381 data = f.read(min(remaining, self.BUFFERSIZE)) 382 remaining -= len(data) 383 else: 384 data = f.read(self.BUFFERSIZE) 385 offset = f.tell() 386 del f 387 388 if leftover: 389 if onlyText: 390 yield leftover[1] 391 else: 392 yield leftover

393

394 - def readlines(self, channel=STDOUT):

395 """Return an iterator that produces newline-terminated lines, 396 excluding header chunks.""" 397 # TODO: make this memory-efficient, by turning it into a generator 398 # that retrieves chunks as necessary, like a pull-driven version of 399 # twisted.protocols.basic.LineReceiver 400 alltext = "".join(self.getChunks([channel], onlyText=True)) 401 io = StringIO(alltext) 402 return io.readlines()

403 413

414 - def unsubscribe(self, receiver):

415 if receiver in self.watchers: 416 self.watchers.remove(receiver)

417

418 - def subscribeConsumer(self, consumer):

419 p = LogFileProducer(self, consumer) 420 p.resumeProducing()

421 422 # interface used by the build steps to add things to the log 423

424 - def merge(self):

425 # merge all .runEntries (which are all of the same type) into a 426 # single chunk for .entries 427 if not self.runEntries: 428 return 429 channel = self.runEntries[0][0] 430 text = "".join([c[1] for c in self.runEntries]) 431 assert channel < 10 432 f = self.openfile 433 f.seek(0, 2) 434 offset = 0 435 while offset < len(text): 436 size = min(len(text)-offset, self.chunkSize) 437 f.write("%d:%d" % (1 + size, channel)) 438 f.write(text[offset:offset+size]) 439 f.write(",") 440 offset += size 441 self.runEntries = [] 442 self.runLength = 0

443

444 - def addEntry(self, channel, text):

445 assert not self.finished 446 447 if isinstance(text, unicode): 448 text = text.encode('utf-8') 449 if channel != HEADER: 450 # Truncate the log if it's more than logMaxSize bytes 451 if self.logMaxSize and self.nonHeaderLength > self.logMaxSize: 452 # Add a message about what's going on 453 if not self.maxLengthExceeded: 454 msg = "\nOutput exceeded %i bytes, remaining output has been truncated\n" % self.logMaxSize 455 self.addEntry(HEADER, msg) 456 self.merge() 457 self.maxLengthExceeded = True 458 459 if self.logMaxTailSize: 460 # Update the tail buffer 461 self.tailBuffer.append((channel, text)) 462 self.tailLength += len(text) 463 while self.tailLength > self.logMaxTailSize: 464 # Drop some stuff off the beginning of the buffer 465 c,t = self.tailBuffer.pop(0) 466 n = len(t) 467 self.tailLength -= n 468 assert self.tailLength >= 0 469 return 470 471 self.nonHeaderLength += len(text) 472 473 # we only add to .runEntries here. merge() is responsible for adding 474 # merged chunks to .entries 475 if self.runEntries and channel != self.runEntries[0][0]: 476 self.merge() 477 self.runEntries.append((channel, text)) 478 self.runLength += len(text) 479 if self.runLength >= self.chunkSize: 480 self.merge() 481 482 for w in self.watchers: 483 w.logChunk(self.step.build, self.step, self, channel, text) 484 self.length += len(text)

485

486 - def addStdout(self, text):

487 self.addEntry(STDOUT, text)

488 - def addStderr(self, text):

489 self.addEntry(STDERR, text)

490 - def addHeader(self, text):

491 self.addEntry(HEADER, text)

492

493 - def finish(self):

494 if self.tailBuffer: 495 msg = "\nFinal %i bytes follow below:\n" % self.tailLength 496 tmp = self.runEntries 497 self.runEntries = [(HEADER, msg)] 498 self.merge() 499 self.runEntries = self.tailBuffer 500 self.merge() 501 self.runEntries = tmp 502 self.merge() 503 self.tailBuffer = [] 504 else: 505 self.merge() 506 507 if self.openfile: 508 # we don't do an explicit close, because there might be readers 509 # shareing the filehandle. As soon as they stop reading, the 510 # filehandle will be released and automatically closed. 511 self.openfile.flush() 512 del self.openfile 513 self.finished = True 514 watchers = self.finishedWatchers 515 self.finishedWatchers = [] 516 for w in watchers: 517 w.callback(self) 518 self.watchers = []

519 520

521 - def compressLog(self):

522 # bail out if there's no compression support 523 if self.compressMethod == "bz2": 524 if BZ2File is None: 525 return 526 compressed = self.getFilename() + ".bz2.tmp" 527 elif self.compressMethod == "gz": 528 if GzipFile is None: 529 return 530 compressed = self.getFilename() + ".gz.tmp" 531 d = threads.deferToThread(self._compressLog, compressed) 532 d.addCallback(self._renameCompressedLog, compressed) 533 d.addErrback(self._cleanupFailedCompress, compressed) 534 return d

535

536 - def _compressLog(self, compressed):

537 infile = self.getFile() 538 if self.compressMethod == "bz2": 539 cf = BZ2File(compressed, 'w') 540 elif self.compressMethod == "gz": 541 cf = GzipFile(compressed, 'w') 542 bufsize = 1024*1024 543 while True: 544 buf = infile.read(bufsize) 545 cf.write(buf) 546 if len(buf) < bufsize: 547 break 548 cf.close()

549 - def _renameCompressedLog(self, rv, compressed):

550 if self.compressMethod == "bz2": 551 filename = self.getFilename() + '.bz2' 552 else: 553 filename = self.getFilename() + '.gz' 554 if sys.platform == 'win32': 555 # windows cannot rename a file on top of an existing one, so 556 # fall back to delete-first. There are ways this can fail and 557 # lose the builder's history, so we avoid using it in the 558 # general (non-windows) case 559 if os.path.exists(filename): 560 os.unlink(filename) 561 os.rename(compressed, filename) 562 _tryremove(self.getFilename(), 1, 5)

563 - def _cleanupFailedCompress(self, failure, compressed):

564 log.msg("failed to compress %s" % self.getFilename()) 565 if os.path.exists(compressed): 566 _tryremove(compressed, 1, 5) 567 failure.trap() # reraise the failure

568 569 # persistence stuff

570 - def __getstate__(self):

571 d = self.__dict__.copy() 572 del d['step'] # filled in upon unpickling 573 del d['watchers'] 574 del d['finishedWatchers'] 575 d['entries'] = [] # let 0.6.4 tolerate the saved log. TODO: really? 576 if d.has_key('finished'): 577 del d['finished'] 578 if d.has_key('openfile'): 579 del d['openfile'] 580 return d

581

582 - def __setstate__(self, d):

583 self.__dict__ = d 584 self.watchers = [] # probably not necessary 585 self.finishedWatchers = [] # same 586 # self.step must be filled in by our parent 587 self.finished = True

588

589 - def upgrade(self, logfilename):

590 """Save our .entries to a new-style offline log file (if necessary), 591 and modify our in-memory representation to use it. The original 592 pickled LogFile (inside the pickled Build) won't be modified.""" 593 self.filename = logfilename 594 if not os.path.exists(self.getFilename()): 595 self.openfile = open(self.getFilename(), "w") 596 self.finished = False 597 for channel,text in self.entries: 598 self.addEntry(channel, text) 599 self.finish() # releases self.openfile, which will be closed 600 del self.entries

601

602 -class HTMLLogFile:

603 implements(interfaces.IStatusLog) 604 605 filename = None 606

607 - def __init__(self, parent, name, logfilename, html):

608 self.step = parent 609 self.name = name 610 self.filename = logfilename 611 self.html = html

612

613 - def getName(self):

614 return self.name # set in BuildStepStatus.addLog

615 - def getStep(self):

616 return self.step

617

618 - def isFinished(self):

619 return True

620 - def waitUntilFinished(self):

621 return defer.succeed(self)

622

623 - def hasContents(self):

624 return True

625 - def getText(self):

626 return self.html # looks kinda like text

627 - def getTextWithHeaders(self):

628 return self.html

629 - def getChunks(self):

630 return [(STDERR, self.html)]

631

634 - def unsubscribe(self, receiver):

635 pass

636

637 - def finish(self):

638 pass

639

640 - def __getstate__(self):

641 d = self.__dict__.copy() 642 del d['step'] 643 return d

644

645 - def upgrade(self, logfilename):

646 pass

647 648

649 -class Event:

650 implements(interfaces.IStatusEvent) 651 652 started = None 653 finished = None 654 text = [] 655 656 # IStatusEvent methods

657 - def getTimes(self):

658 return (self.started, self.finished)

659 - def getText(self):

660 return self.text

661 - def getLogs(self):

662 return []

663

664 - def finish(self):

665 self.finished = util.now()

666

667 -class TestResult:

668 implements(interfaces.ITestResult) 669

670 - def __init__(self, name, results, text, logs):

671 assert isinstance(name, tuple) 672 self.name = name 673 self.results = results 674 self.text = text 675 self.logs = logs

676

677 - def getName(self):

678 return self.name

679

680 - def getResults(self):

681 return self.results

682

683 - def getText(self):

684 return self.text

685

686 - def getLogs(self):

687 return self.logs

688 689

690 -class BuildSetStatus:

691 implements(interfaces.IBuildSetStatus) 692

693 - def __init__(self, source, reason, builderNames, bsid=None):

694 self.source = source 695 self.reason = reason 696 self.builderNames = builderNames 697 self.id = bsid 698 self.successWatchers = [] 699 self.finishedWatchers = [] 700 self.stillHopeful = True 701 self.finished = False

702

703 - def setBuildRequestStatuses(self, buildRequestStatuses):

704 self.buildRequests = buildRequestStatuses

705 - def setResults(self, results):

706 # the build set succeeds only if all its component builds succeed 707 self.results = results

708 - def giveUpHope(self):

709 self.stillHopeful = False

710 711

712 - def notifySuccessWatchers(self):

713 for d in self.successWatchers: 714 d.callback(self) 715 self.successWatchers = []

716

717 - def notifyFinishedWatchers(self):

718 self.finished = True 719 for d in self.finishedWatchers: 720 d.callback(self) 721 self.finishedWatchers = []

722 723 # methods for our clients 724

725 - def getSourceStamp(self):

726 return self.source

727 - def getReason(self):

728 return self.reason

729 - def getResults(self):

730 return self.results

731 - def getID(self):

732 return self.id

733

734 - def getBuilderNames(self):

735 return self.builderNames

736 - def getBuildRequests(self):

737 return self.buildRequests

738 - def isFinished(self):

739 return self.finished

740

741 - def waitUntilSuccess(self):

742 if self.finished or not self.stillHopeful: 743 # the deferreds have already fired 744 return defer.succeed(self) 745 d = defer.Deferred() 746 self.successWatchers.append(d) 747 return d

748

749 - def waitUntilFinished(self):

750 if self.finished: 751 return defer.succeed(self) 752 d = defer.Deferred() 753 self.finishedWatchers.append(d) 754 return d

755

756 -class BuildRequestStatus:

757 implements(interfaces.IBuildRequestStatus) 758

759 - def __init__(self, source, builderName):

760 self.source = source 761 self.builderName = builderName 762 self.builds = [] # list of BuildStatus objects 763 self.observers = [] 764 self.submittedAt = None

765

766 - def buildStarted(self, build):

767 self.builds.append(build) 768 for o in self.observers[:]: 769 o(build)

770 771 # methods called by our clients

772 - def getSourceStamp(self):

773 return self.source

774 - def getBuilderName(self):

775 return self.builderName

776 - def getBuilds(self):

777 return self.builds

778

783 - def unsubscribe(self, observer):

784 self.observers.remove(observer)

785

786 - def getSubmitTime(self):

787 return self.submittedAt

788 - def setSubmitTime(self, t):

789 self.submittedAt = t

790 791

792 -class BuildStepStatus(styles.Versioned):

793 """ 794 I represent a collection of output status for a 795 L{buildbot.process.step.BuildStep}. 796 797 Statistics contain any information gleaned from a step that is 798 not in the form of a logfile. As an example, steps that run 799 tests might gather statistics about the number of passed, failed, 800 or skipped tests. 801 802 @type progress: L{buildbot.status.progress.StepProgress} 803 @cvar progress: tracks ETA for the step 804 @type text: list of strings 805 @cvar text: list of short texts that describe the command and its status 806 @type text2: list of strings 807 @cvar text2: list of short texts added to the overall build description 808 @type logs: dict of string -> L{buildbot.status.builder.LogFile} 809 @ivar logs: logs of steps 810 @type statistics: dict 811 @ivar statistics: results from running this step 812 """ 813 # note that these are created when the Build is set up, before each 814 # corresponding BuildStep has started. 815 implements(interfaces.IBuildStepStatus, interfaces.IStatusEvent) 816 persistenceVersion = 2 817 818 started = None 819 finished = None 820 progress = None 821 text = [] 822 results = (None, []) 823 text2 = [] 824 watchers = [] 825 updates = {} 826 finishedWatchers = [] 827 statistics = {} 828

829 - def __init__(self, parent):

830 assert interfaces.IBuildStatus(parent) 831 self.build = parent 832 self.logs = [] 833 self.urls = {} 834 self.watchers = [] 835 self.updates = {} 836 self.finishedWatchers = [] 837 self.statistics = {}

838

839 - def getName(self):

840 """Returns a short string with the name of this step. This string 841 may have spaces in it.""" 842 return self.name

843

844 - def getBuild(self):

845 return self.build

846

847 - def getTimes(self):

848 return (self.started, self.finished)

849

850 - def getExpectations(self):

851 """Returns a list of tuples (name, current, target).""" 852 if not self.progress: 853 return [] 854 ret = [] 855 metrics = self.progress.progress.keys() 856 metrics.sort() 857 for m in metrics: 858 t = (m, self.progress.progress[m], self.progress.expectations[m]) 859 ret.append(t) 860 return ret

861

862 - def getLogs(self):

863 return self.logs

864

865 - def getURLs(self):

866 return self.urls.copy()

867

868 - def isStarted(self):

869 return (self.started is not None)

870

871 - def isFinished(self):

872 return (self.finished is not None)

873

874 - def waitUntilFinished(self):

875 if self.finished: 876 d = defer.succeed(self) 877 else: 878 d = defer.Deferred() 879 self.finishedWatchers.append(d) 880 return d

881 882 # while the step is running, the following methods make sense. 883 # Afterwards they return None 884

885 - def getETA(self):

886 if self.started is None: 887 return None # not started yet 888 if self.finished is not None: 889 return None # already finished 890 if not self.progress: 891 return None # no way to predict 892 return self.progress.remaining()

893 894 # Once you know the step has finished, the following methods are legal. 895 # Before this step has finished, they all return None. 896

897 - def getText(self):

898 """Returns a list of strings which describe the step. These are 899 intended to be displayed in a narrow column. If more space is 900 available, the caller should join them together with spaces before 901 presenting them to the user.""" 902 return self.text

903

904 - def getResults(self):

905 """Return a tuple describing the results of the step. 906 'result' is one of the constants in L{buildbot.status.builder}: 907 SUCCESS, WARNINGS, FAILURE, or SKIPPED. 908 'strings' is an optional list of strings that the step wants to 909 append to the overall build's results. These strings are usually 910 more terse than the ones returned by getText(): in particular, 911 successful Steps do not usually contribute any text to the 912 overall build. 913 914 @rtype: tuple of int, list of strings 915 @returns: (result, strings) 916 """ 917 return (self.results, self.text2)

918

919 - def hasStatistic(self, name):

920 """Return true if this step has a value for the given statistic. 921 """ 922 return self.statistics.has_key(name)

923

924 - def getStatistic(self, name, default=None):

925 """Return the given statistic, if present 926 """ 927 return self.statistics.get(name, default)

928 929 # subscription interface 930 936

937 - def sendETAUpdate(self, receiver, updateInterval):

938 self.updates[receiver] = None 939 # they might unsubscribe during stepETAUpdate 940 receiver.stepETAUpdate(self.build, self, 941 self.getETA(), self.getExpectations()) 942 if receiver in self.watchers: 943 self.updates[receiver] = reactor.callLater(updateInterval, 944 self.sendETAUpdate, 945 receiver, 946 updateInterval)

947

948 - def unsubscribe(self, receiver):

949 if receiver in self.watchers: 950 self.watchers.remove(receiver) 951 if receiver in self.updates: 952 if self.updates[receiver] is not None: 953 self.updates[receiver].cancel() 954 del self.updates[receiver]

955 956 957 # methods to be invoked by the BuildStep 958

959 - def setName(self, stepname):

960 self.name = stepname

961

962 - def setColor(self, color):

963 log.msg("BuildStepStatus.setColor is no longer supported -- ignoring color %s" % (color,))

964

965 - def setProgress(self, stepprogress):

966 self.progress = stepprogress

967

968 - def stepStarted(self):

969 self.started = util.now() 970 if self.build: 971 self.build.stepStarted(self)

972

973 - def addLog(self, name):

974 assert self.started # addLog before stepStarted won't notify watchers 975 logfilename = self.build.generateLogfileName(self.name, name) 976 log = LogFile(self, name, logfilename) 977 log.logMaxSize = self.build.builder.logMaxSize 978 log.logMaxTailSize = self.build.builder.logMaxTailSize 979 log.compressMethod = self.build.builder.logCompressionMethod 980 self.logs.append(log) 981 for w in self.watchers: 982 receiver = w.logStarted(self.build, self, log) 983 if receiver: 984 log.subscribe(receiver, True) 985 d = log.waitUntilFinished() 986 d.addCallback(lambda log: log.unsubscribe(receiver)) 987 d = log.waitUntilFinished() 988 d.addCallback(self.logFinished) 989 return log

990

991 - def addHTMLLog(self, name, html):

992 assert self.started # addLog before stepStarted won't notify watchers 993 logfilename = self.build.generateLogfileName(self.name, name) 994 log = HTMLLogFile(self, name, logfilename, html) 995 self.logs.append(log) 996 for w in self.watchers: 997 receiver = w.logStarted(self.build, self, log) 998 # TODO: think about this: there isn't much point in letting 999 # them subscribe 1000 #if receiver: 1001 # log.subscribe(receiver, True) 1002 w.logFinished(self.build, self, log)

1003

1004 - def logFinished(self, log):

1005 for w in self.watchers: 1006 w.logFinished(self.build, self, log)

1007

1008 - def addURL(self, name, url):

1009 self.urls[name] = url

1010

1011 - def setText(self, text):

1012 self.text = text 1013 for w in self.watchers: 1014 w.stepTextChanged(self.build, self, text)

1015 - def setText2(self, text):

1016 self.text2 = text 1017 for w in self.watchers: 1018 w.stepText2Changed(self.build, self, text)

1019

1020 - def setStatistic(self, name, value):

1021 """Set the given statistic. Usually called by subclasses. 1022 """ 1023 self.statistics[name] = value

1024

1025 - def stepFinished(self, results):

1026 self.finished = util.now() 1027 self.results = results 1028 cld = [] # deferreds for log compression 1029 logCompressionLimit = self.build.builder.logCompressionLimit 1030 for loog in self.logs: 1031 if not loog.isFinished(): 1032 loog.finish() 1033 # if log compression is on, and it's a real LogFile, 1034 # HTMLLogFiles aren't files 1035 if logCompressionLimit is not False and \ 1036 isinstance(loog, LogFile): 1037 if os.path.getsize(loog.getFilename()) > logCompressionLimit: 1038 loog_deferred = loog.compressLog() 1039 if loog_deferred: 1040 cld.append(loog_deferred) 1041 1042 for r in self.updates.keys(): 1043 if self.updates[r] is not None: 1044 self.updates[r].cancel() 1045 del self.updates[r] 1046 1047 watchers = self.finishedWatchers 1048 self.finishedWatchers = [] 1049 for w in watchers: 1050 w.callback(self) 1051 if cld: 1052 return defer.DeferredList(cld)

1053

1054 - def checkLogfiles(self):

1055 # filter out logs that have been deleted 1056 self.logs = [ l for l in self.logs if l.hasContents() ]

1057 1058 # persistence 1059

1060 - def __getstate__(self):

1061 d = styles.Versioned.__getstate__(self) 1062 del d['build'] # filled in when loading 1063 if d.has_key('progress'): 1064 del d['progress'] 1065 del d['watchers'] 1066 del d['finishedWatchers'] 1067 del d['updates'] 1068 return d

1069

1070 - def __setstate__(self, d):

1071 styles.Versioned.__setstate__(self, d) 1072 # self.build must be filled in by our parent 1073 1074 # point the logs to this object 1075 for loog in self.logs: 1076 loog.step = self

1077

1078 - def upgradeToVersion1(self):

1079 if not hasattr(self, "urls"): 1080 self.urls = {}

1081

1082 - def upgradeToVersion2(self):

1083 if not hasattr(self, "statistics"): 1084 self.statistics = {}

1085 1086

1087 -class BuildStatus(styles.Versioned):

1088 implements(interfaces.IBuildStatus, interfaces.IStatusEvent) 1089 persistenceVersion = 3 1090 1091 source = None 1092 reason = None 1093 changes = [] 1094 blamelist = [] 1095 requests = [] 1096 progress = None 1097 started = None 1098 finished = None 1099 currentStep = None 1100 text = [] 1101 results = None 1102 slavename = "???" 1103 1104 # these lists/dicts are defined here so that unserialized instances have 1105 # (empty) values. They are set in __init__ to new objects to make sure 1106 # each instance gets its own copy. 1107 watchers = [] 1108 updates = {} 1109 finishedWatchers = [] 1110 testResults = {} 1111

1112 - def __init__(self, parent, number):

1113 """ 1114 @type parent: L{BuilderStatus} 1115 @type number: int 1116 """ 1117 assert interfaces.IBuilderStatus(parent) 1118 self.builder = parent 1119 self.number = number 1120 self.watchers = [] 1121 self.updates = {} 1122 self.finishedWatchers = [] 1123 self.steps = [] 1124 self.testResults = {} 1125 self.properties = Properties() 1126 self.requests = []

1127

1128 - def __repr__(self):

1129 return "<%s #%s>" % (self.__class__.__name__, self.number)

1130 1131 # IBuildStatus 1132

1133 - def getBuilder(self):

1134 """ 1135 @rtype: L{BuilderStatus} 1136 """ 1137 return self.builder

1138

1139 - def getProperty(self, propname):

1140 return self.properties[propname]

1141

1142 - def getProperties(self):

1143 return self.properties

1144

1145 - def getNumber(self):

1146 return self.number

1147

1148 - def getPreviousBuild(self):

1149 if self.number == 0: 1150 return None 1151 return self.builder.getBuild(self.number-1)

1152

1153 - def getSourceStamp(self, absolute=False):

1154 if not absolute or not self.properties.has_key('got_revision'): 1155 return self.source 1156 return self.source.getAbsoluteSourceStamp(self.properties['got_revision'])

1157

1158 - def getReason(self):

1159 return self.reason

1160

1161 - def getChanges(self):

1162 return self.changes

1163

1164 - def getRequests(self):

1165 return self.requests

1166

1167 - def getResponsibleUsers(self):

1168 return self.blamelist

1169

1170 - def getInterestedUsers(self):

1171 # TODO: the Builder should add others: sheriffs, domain-owners 1172 return self.blamelist + self.properties.getProperty('owners', [])

1173

1174 - def getSteps(self):

1175 """Return a list of IBuildStepStatus objects. For invariant builds 1176 (those which always use the same set of Steps), this should be the 1177 complete list, however some of the steps may not have started yet 1178 (step.getTimes()[0] will be None). For variant builds, this may not 1179 be complete (asking again later may give you more of them).""" 1180 return self.steps

1181

1182 - def getTimes(self):

1183 return (self.started, self.finished)

1184 1185 _sentinel = [] # used as a sentinel to indicate unspecified initial_value

1186 - def getSummaryStatistic(self, name, summary_fn, initial_value=_sentinel):

1187 """Summarize the named statistic over all steps in which it 1188 exists, using combination_fn and initial_value to combine multiple 1189 results into a single result. This translates to a call to Python's 1190 X{reduce}:: 1191 return reduce(summary_fn, step_stats_list, initial_value) 1192 """ 1193 step_stats_list = [ 1194 st.getStatistic(name) 1195 for st in self.steps 1196 if st.hasStatistic(name) ] 1197 if initial_value is self._sentinel: 1198 return reduce(summary_fn, step_stats_list) 1199 else: 1200 return reduce(summary_fn, step_stats_list, initial_value)

1201

1202 - def isFinished(self):

1203 return (self.finished is not None)

1204

1205 - def waitUntilFinished(self):

1206 if self.finished: 1207 d = defer.succeed(self) 1208 else: 1209 d = defer.Deferred() 1210 self.finishedWatchers.append(d) 1211 return d

1212 1213 # while the build is running, the following methods make sense. 1214 # Afterwards they return None 1215

1216 - def getETA(self):

1217 if self.finished is not None: 1218 return None 1219 if not self.progress: 1220 return None 1221 eta = self.progress.eta() 1222 if eta is None: 1223 return None 1224 return eta - util.now()

1225

1226 - def getCurrentStep(self):

1227 return self.currentStep

1228 1229 # Once you know the build has finished, the following methods are legal. 1230 # Before ths build has finished, they all return None. 1231

1232 - def getText(self):

1233 text = [] 1234 text.extend(self.text) 1235 for s in self.steps: 1236 text.extend(s.text2) 1237 return text

1238

1239 - def getResults(self):

1240 return self.results

1241

1242 - def getSlavename(self):

1243 return self.slavename

1244

1245 - def getTestResults(self):

1246 return self.testResults

1247

1248 - def getLogs(self):

1249 # TODO: steps should contribute significant logs instead of this 1250 # hack, which returns every log from every step. The logs should get 1251 # names like "compile" and "test" instead of "compile.output" 1252 logs = [] 1253 for s in self.steps: 1254 for log in s.getLogs(): 1255 logs.append(log) 1256 return logs

1257 1258 # subscription interface 1259 1266

1267 - def sendETAUpdate(self, receiver, updateInterval):

1268 self.updates[receiver] = None 1269 ETA = self.getETA() 1270 if ETA is not None: 1271 receiver.buildETAUpdate(self, self.getETA()) 1272 # they might have unsubscribed during buildETAUpdate 1273 if receiver in self.watchers: 1274 self.updates[receiver] = reactor.callLater(updateInterval, 1275 self.sendETAUpdate, 1276 receiver, 1277 updateInterval)

1278

1279 - def unsubscribe(self, receiver):

1280 if receiver in self.watchers: 1281 self.watchers.remove(receiver) 1282 if receiver in self.updates: 1283 if self.updates[receiver] is not None: 1284 self.updates[receiver].cancel() 1285 del self.updates[receiver]

1286 1287 # methods for the base.Build to invoke 1288

1289 - def addStepWithName(self, name):

1290 """The Build is setting up, and has added a new BuildStep to its 1291 list. Create a BuildStepStatus object to which it can send status 1292 updates.""" 1293 1294 s = BuildStepStatus(self) 1295 s.setName(name) 1296 self.steps.append(s) 1297 return s

1298

1299 - def setProperty(self, propname, value, source):

1300 self.properties.setProperty(propname, value, source)

1301

1302 - def addTestResult(self, result):

1303 self.testResults[result.getName()] = result

1304

1305 - def setSourceStamp(self, sourceStamp):

1306 self.source = sourceStamp 1307 self.changes = self.source.changes

1308

1309 - def setRequests(self, requests):

1310 self.requests = requests

1311

1312 - def setReason(self, reason):

1313 self.reason = reason

1314 - def setBlamelist(self, blamelist):

1315 self.blamelist = blamelist

1316 - def setProgress(self, progress):

1317 self.progress = progress

1318

1319 - def buildStarted(self, build):

1320 """The Build has been set up and is about to be started. It can now 1321 be safely queried, so it is time to announce the new build.""" 1322 1323 self.started = util.now() 1324 # now that we're ready to report status, let the BuilderStatus tell 1325 # the world about us 1326 self.builder.buildStarted(self)

1327

1328 - def setSlavename(self, slavename):

1329 self.slavename = slavename

1330

1331 - def setText(self, text):

1332 assert isinstance(text, (list, tuple)) 1333 self.text = text

1334 - def setResults(self, results):

1335 self.results = results

1336

1337 - def buildFinished(self):

1338 self.currentStep = None 1339 self.finished = util.now() 1340 1341 for r in self.updates.keys(): 1342 if self.updates[r] is not None: 1343 self.updates[r].cancel() 1344 del self.updates[r] 1345 1346 watchers = self.finishedWatchers 1347 self.finishedWatchers = [] 1348 for w in watchers: 1349 w.callback(self)

1350 1351 # methods called by our BuildStepStatus children 1352

1353 - def stepStarted(self, step):

1354 self.currentStep = step 1355 name = self.getBuilder().getName() 1356 for w in self.watchers: 1357 receiver = w.stepStarted(self, step) 1358 if receiver: 1359 if type(receiver) == type(()): 1360 step.subscribe(receiver[0], receiver[1]) 1361 else: 1362 step.subscribe(receiver) 1363 d = step.waitUntilFinished() 1364 d.addCallback(lambda step: step.unsubscribe(receiver)) 1365 1366 step.waitUntilFinished().addCallback(self._stepFinished)

1367

1368 - def _stepFinished(self, step):

1369 results = step.getResults() 1370 for w in self.watchers: 1371 w.stepFinished(self, step, results)

1372 1373 # methods called by our BuilderStatus parent 1374

1375 - def pruneSteps(self):

1376 # this build is very old: remove the build steps too 1377 self.steps = []

1378 1379 # persistence stuff 1380

1381 - def generateLogfileName(self, stepname, logname):

1382 """Return a filename (relative to the Builder's base directory) where 1383 the logfile's contents can be stored uniquely. 1384 1385 The base filename is made by combining our build number, the Step's 1386 name, and the log's name, then removing unsuitable characters. The 1387 filename is then made unique by appending _0, _1, etc, until it does 1388 not collide with any other logfile. 1389 1390 These files are kept in the Builder's basedir (rather than a 1391 per-Build subdirectory) because that makes cleanup easier: cron and 1392 find will help get rid of the old logs, but the empty directories are 1393 more of a hassle to remove.""" 1394 1395 starting_filename = "%d-log-%s-%s" % (self.number, stepname, logname) 1396 starting_filename = re.sub(r'[^\w\.\-]', '_', starting_filename) 1397 # now make it unique 1398 unique_counter = 0 1399 filename = starting_filename 1400 while filename in [l.filename 1401 for step in self.steps 1402 for l in step.getLogs() 1403 if l.filename]: 1404 filename = "%s_%d" % (starting_filename, unique_counter) 1405 unique_counter += 1 1406 return filename

1407

1408 - def __getstate__(self):

1409 d = styles.Versioned.__getstate__(self) 1410 # for now, a serialized Build is always "finished". We will never 1411 # save unfinished builds. 1412 if not self.finished: 1413 d['finished'] = True 1414 # TODO: push an "interrupted" step so it is clear that the build 1415 # was interrupted. The builder will have a 'shutdown' event, but 1416 # someone looking at just this build will be confused as to why 1417 # the last log is truncated. 1418 for k in 'builder', 'watchers', 'updates', 'requests', 'finishedWatchers': 1419 if k in d: del d[k] 1420 return d

1421

1422 - def __setstate__(self, d):

1423 styles.Versioned.__setstate__(self, d) 1424 # self.builder must be filled in by our parent when loading 1425 for step in self.steps: 1426 step.build = self 1427 self.watchers = [] 1428 self.updates = {} 1429 self.finishedWatchers = []

1430

1431 - def upgradeToVersion1(self):

1432 if hasattr(self, "sourceStamp"): 1433 # the old .sourceStamp attribute wasn't actually very useful 1434 maxChangeNumber, patch = self.sourceStamp 1435 changes = getattr(self, 'changes', []) 1436 source = sourcestamp.SourceStamp(branch=None, 1437 revision=None, 1438 patch=patch, 1439 changes=changes) 1440 self.source = source 1441 self.changes = source.changes 1442 del self.sourceStamp

1443

1444 - def upgradeToVersion2(self):

1445 self.properties = {}

1446

1447 - def upgradeToVersion3(self):

1448 # in version 3, self.properties became a Properties object 1449 propdict = self.properties 1450 self.properties = Properties() 1451 self.properties.update(propdict, "Upgrade from previous version")

1452

1453 - def upgradeLogfiles(self):

1454 # upgrade any LogFiles that need it. This must occur after we've been 1455 # attached to our Builder, and after we know about all LogFiles of 1456 # all Steps (to get the filenames right). 1457 assert self.builder 1458 for s in self.steps: 1459 for l in s.getLogs(): 1460 if l.filename: 1461 pass # new-style, log contents are on disk 1462 else: 1463 logfilename = self.generateLogfileName(s.name, l.name) 1464 # let the logfile update its .filename pointer, 1465 # transferring its contents onto disk if necessary 1466 l.upgrade(logfilename)

1467

1468 - def checkLogfiles(self):

1469 # check that all logfiles exist, and remove references to any that 1470 # have been deleted (e.g., by purge()) 1471 for s in self.steps: 1472 s.checkLogfiles()

1473

1474 - def saveYourself(self):

1475 filename = os.path.join(self.builder.basedir, "%d" % self.number) 1476 if os.path.isdir(filename): 1477 # leftover from 0.5.0, which stored builds in directories 1478 shutil.rmtree(filename, ignore_errors=True) 1479 tmpfilename = filename + ".tmp" 1480 try: 1481 dump(self, open(tmpfilename, "wb"), -1) 1482 if sys.platform == 'win32': 1483 # windows cannot rename a file on top of an existing one, so 1484 # fall back to delete-first. There are ways this can fail and 1485 # lose the builder's history, so we avoid using it in the 1486 # general (non-windows) case 1487 if os.path.exists(filename): 1488 os.unlink(filename) 1489 os.rename(tmpfilename, filename) 1490 except: 1491 log.msg("unable to save build %s-#%d" % (self.builder.name, 1492 self.number)) 1493 log.err()

1494 1495 1496

1497 -class BuilderStatus(styles.Versioned):

1498 """I handle status information for a single process.base.Builder object. 1499 That object sends status changes to me (frequently as Events), and I 1500 provide them on demand to the various status recipients, like the HTML 1501 waterfall display and the live status clients. It also sends build 1502 summaries to me, which I log and provide to status clients who aren't 1503 interested in seeing details of the individual build steps. 1504 1505 I am responsible for maintaining the list of historic Events and Builds, 1506 pruning old ones, and loading them from / saving them to disk. 1507 1508 I live in the buildbot.process.base.Builder object, in the 1509 .builder_status attribute. 1510 1511 @type category: string 1512 @ivar category: user-defined category this builder belongs to; can be 1513 used to filter on in status clients 1514 """ 1515 1516 implements(interfaces.IBuilderStatus, interfaces.IEventSource) 1517 persistenceVersion = 1 1518 1519 # these limit the amount of memory we consume, as well as the size of the 1520 # main Builder pickle. The Build and LogFile pickles on disk must be 1521 # handled separately. 1522 buildCacheSize = 15 1523 eventHorizon = 50 # forget events beyond this 1524 1525 # these limit on-disk storage 1526 logHorizon = 40 # forget logs in steps in builds beyond this 1527 buildHorizon = 100 # forget builds beyond this 1528 1529 category = None 1530 currentBigState = "offline" # or idle/waiting/interlocked/building 1531 basedir = None # filled in by our parent 1532

1533 - def __init__(self, buildername, category=None):

1534 self.name = buildername 1535 self.category = category 1536 1537 self.slavenames = [] 1538 self.events = [] 1539 # these three hold Events, and are used to retrieve the current 1540 # state of the boxes. 1541 self.lastBuildStatus = None 1542 #self.currentBig = None 1543 #self.currentSmall = None 1544 self.currentBuilds = [] 1545 self.pendingBuilds = [] 1546 self.nextBuild = None 1547 self.watchers = [] 1548 self.buildCache = weakref.WeakValueDictionary() 1549 self.buildCache_LRU = [] 1550 self.logCompressionLimit = False # default to no compression for tests 1551 self.logCompressionMethod = "bz2" 1552 self.logMaxSize = None # No default limit 1553 self.logMaxTailSize = None # No tail buffering

1554 1555 # persistence 1556

1557 - def __getstate__(self):

1558 # when saving, don't record transient stuff like what builds are 1559 # currently running, because they won't be there when we start back 1560 # up. Nor do we save self.watchers, nor anything that gets set by our 1561 # parent like .basedir and .status 1562 d = styles.Versioned.__getstate__(self) 1563 d['watchers'] = [] 1564 del d['buildCache'] 1565 del d['buildCache_LRU'] 1566 for b in self.currentBuilds: 1567 b.saveYourself() 1568 # TODO: push a 'hey, build was interrupted' event 1569 del d['currentBuilds'] 1570 del d['pendingBuilds'] 1571 del d['currentBigState'] 1572 del d['basedir'] 1573 del d['status'] 1574 del d['nextBuildNumber'] 1575 return d

1576

1577 - def __setstate__(self, d):

1578 # when loading, re-initialize the transient stuff. Remember that 1579 # upgradeToVersion1 and such will be called after this finishes. 1580 styles.Versioned.__setstate__(self, d) 1581 self.buildCache = weakref.WeakValueDictionary() 1582 self.buildCache_LRU = [] 1583 self.currentBuilds = [] 1584 self.pendingBuilds = [] 1585 self.watchers = [] 1586 self.slavenames = []

1587 # self.basedir must be filled in by our parent 1588 # self.status must be filled in by our parent 1589

1590 - def reconfigFromBuildmaster(self, buildmaster):

1591 # Note that we do not hang onto the buildmaster, since this object 1592 # gets pickled and unpickled. 1593 if buildmaster.buildCacheSize: 1594 self.buildCacheSize = buildmaster.buildCacheSize 1595 if buildmaster.eventHorizon: 1596 self.eventHorizon = buildmaster.eventHorizon 1597 if buildmaster.logHorizon: 1598 self.logHorizon = buildmaster.logHorizon 1599 if buildmaster.buildHorizon: 1600 self.buildHorizon = buildmaster.buildHorizon

1601

1602 - def upgradeToVersion1(self):

1603 if hasattr(self, 'slavename'): 1604 self.slavenames = [self.slavename] 1605 del self.slavename 1606 if hasattr(self, 'nextBuildNumber'): 1607 del self.nextBuildNumber # determineNextBuildNumber chooses this

1608

1609 - def determineNextBuildNumber(self):

1610 """Scan our directory of saved BuildStatus instances to determine 1611 what our self.nextBuildNumber should be. Set it one larger than the 1612 highest-numbered build we discover. This is called by the top-level 1613 Status object shortly after we are created or loaded from disk. 1614 """ 1615 existing_builds = [int(f) 1616 for f in os.listdir(self.basedir) 1617 if re.match("^\d+$", f)] 1618 if existing_builds: 1619 self.nextBuildNumber = max(existing_builds) + 1 1620 else: 1621 self.nextBuildNumber = 0

1622

1623 - def setLogCompressionLimit(self, lowerLimit):

1624 self.logCompressionLimit = lowerLimit

1625

1626 - def setLogCompressionMethod(self, method):

1627 assert method in ("bz2", "gz") 1628 self.logCompressionMethod = method

1629

1630 - def setLogMaxSize(self, upperLimit):

1631 self.logMaxSize = upperLimit

1632

1633 - def setLogMaxTailSize(self, tailSize):

1634 self.logMaxTailSize = tailSize

1635

1636 - def saveYourself(self):

1637 for b in self.currentBuilds: 1638 if not b.isFinished: 1639 # interrupted build, need to save it anyway. 1640 # BuildStatus.saveYourself will mark it as interrupted. 1641 b.saveYourself() 1642 filename = os.path.join(self.basedir, "builder") 1643 tmpfilename = filename + ".tmp" 1644 try: 1645 dump(self, open(tmpfilename, "wb"), -1) 1646 if sys.platform == 'win32': 1647 # windows cannot rename a file on top of an existing one 1648 if os.path.exists(filename): 1649 os.unlink(filename) 1650 os.rename(tmpfilename, filename) 1651 except: 1652 log.msg("unable to save builder %s" % self.name) 1653 log.err()

1654 1655 1656 # build cache management 1657

1658 - def makeBuildFilename(self, number):

1659 return os.path.join(self.basedir, "%d" % number)

1660

1661 - def touchBuildCache(self, build):

1662 self.buildCache[build.number] = build 1663 if build in self.buildCache_LRU: 1664 self.buildCache_LRU.remove(build) 1665 self.buildCache_LRU = self.buildCache_LRU[-(self.buildCacheSize-1):] + [ build ] 1666 return build

1667

1668 - def getBuildByNumber(self, number):

1669 # first look in currentBuilds 1670 for b in self.currentBuilds: 1671 if b.number == number: 1672 return self.touchBuildCache(b) 1673 1674 # then in the buildCache 1675 if number in self.buildCache: 1676 return self.touchBuildCache(self.buildCache[number]) 1677 1678 # then fall back to loading it from disk 1679 filename = self.makeBuildFilename(number) 1680 try: 1681 log.msg("Loading builder %s's build %d from on-disk pickle" 1682 % (self.name, number)) 1683 build = load(open(filename, "rb")) 1684 styles.doUpgrade() 1685 build.builder = self 1686 # handle LogFiles from after 0.5.0 and before 0.6.5 1687 build.upgradeLogfiles() 1688 # check that logfiles exist 1689 build.checkLogfiles() 1690 return self.touchBuildCache(build) 1691 except IOError: 1692 raise IndexError("no such build %d" % number) 1693 except EOFError: 1694 raise IndexError("corrupted build pickle %d" % number)

1695

1696 - def prune(self):

1697 gc.collect() 1698 1699 # begin by pruning our own events 1700 self.events = self.events[-self.eventHorizon:] 1701 1702 # get the horizons straight 1703 if self.buildHorizon: 1704 earliest_build = self.nextBuildNumber - self.buildHorizon 1705 else: 1706 earliest_build = 0 1707 1708 if self.logHorizon: 1709 earliest_log = self.nextBuildNumber - self.logHorizon 1710 else: 1711 earliest_log = 0 1712 1713 if earliest_log < earliest_build: 1714 earliest_log = earliest_build 1715 1716 if earliest_build == 0: 1717 return 1718 1719 # skim the directory and delete anything that shouldn't be there anymore 1720 build_re = re.compile(r"^([0-9]+)$") 1721 build_log_re = re.compile(r"^([0-9]+)-.*$") 1722 # if the directory doesn't exist, bail out here 1723 if not os.path.exists(self.basedir): 1724 return 1725 1726 for filename in os.listdir(self.basedir): 1727 num = None 1728 mo = build_re.match(filename) 1729 is_logfile = False 1730 if mo: 1731 num = int(mo.group(1)) 1732 else: 1733 mo = build_log_re.match(filename) 1734 if mo: 1735 num = int(mo.group(1)) 1736 is_logfile = True 1737 1738 if num is None: continue 1739 if num in self.buildCache: continue 1740 1741 if (is_logfile and num < earliest_log) or num < earliest_build: 1742 pathname = os.path.join(self.basedir, filename) 1743 log.msg("pruning '%s'" % pathname) 1744 try: os.unlink(pathname) 1745 except OSError: pass

1746 1747 # IBuilderStatus methods

1748 - def getName(self):

1749 return self.name

1750

1751 - def getState(self):

1752 return (self.currentBigState, self.currentBuilds)

1753

1754 - def getSlaves(self):

1755 return [self.status.getSlave(name) for name in self.slavenames]

1756

1757 - def getPendingBuilds(self):

1758 return self.pendingBuilds

1759

1760 - def getCurrentBuilds(self):

1761 return self.currentBuilds

1762

1763 - def getLastFinishedBuild(self):

1764 b = self.getBuild(-1) 1765 if not (b and b.isFinished()): 1766 b = self.getBuild(-2) 1767 return b

1768

1769 - def getCategory(self):

1770 return self.category

1771

1772 - def getBuild(self, number):

1773 if number < 0: 1774 number = self.nextBuildNumber + number 1775 if number < 0 or number >= self.nextBuildNumber: 1776 return None 1777 1778 try: 1779 return self.getBuildByNumber(number) 1780 except IndexError: 1781 return None

1782

1783 - def getEvent(self, number):

1784 try: 1785 return self.events[number] 1786 except IndexError: 1787 return None

1788

1789 - def generateFinishedBuilds(self, branches=[], 1790 num_builds=None, 1791 max_buildnum=None, 1792 finished_before=None, 1793 max_search=200):

1794 got = 0 1795 for Nb in itertools.count(1): 1796 if Nb > self.nextBuildNumber: 1797 break 1798 if Nb > max_search: 1799 break 1800 build = self.getBuild(-Nb) 1801 if build is None: 1802 continue 1803 if max_buildnum is not None: 1804 if build.getNumber() > max_buildnum: 1805 continue 1806 if not build.isFinished(): 1807 continue 1808 if finished_before is not None: 1809 start, end = build.getTimes() 1810 if end >= finished_before: 1811 continue 1812 if branches: 1813 if build.getSourceStamp().branch not in branches: 1814 continue 1815 got += 1 1816 yield build 1817 if num_builds is not None: 1818 if got >= num_builds: 1819 return

1820

1821 - def eventGenerator(self, branches=[], categories=[]):

1822 """This function creates a generator which will provide all of this 1823 Builder's status events, starting with the most recent and 1824 progressing backwards in time. """ 1825 1826 # remember the oldest-to-earliest flow here. "next" means earlier. 1827 1828 # TODO: interleave build steps and self.events by timestamp. 1829 # TODO: um, I think we're already doing that. 1830 1831 # TODO: there's probably something clever we could do here to 1832 # interleave two event streams (one from self.getBuild and the other 1833 # from self.getEvent), which would be simpler than this control flow 1834 1835 eventIndex = -1 1836 e = self.getEvent(eventIndex) 1837 for Nb in range(1, self.nextBuildNumber+1): 1838 b = self.getBuild(-Nb) 1839 if not b: 1840 # HACK: If this is the first build we are looking at, it is 1841 # possible it's in progress but locked before it has written a 1842 # pickle; in this case keep looking. 1843 if Nb == 1: 1844 continue 1845 break 1846 if branches and not b.getSourceStamp().branch in branches: 1847 continue 1848 if categories and not b.getBuilder().getCategory() in categories: 1849 continue 1850 steps = b.getSteps() 1851 for Ns in range(1, len(steps)+1): 1852 if steps[-Ns].started: 1853 step_start = steps[-Ns].getTimes()[0] 1854 while e is not None and e.getTimes()[0] > step_start: 1855 yield e 1856 eventIndex -= 1 1857 e = self.getEvent(eventIndex) 1858 yield steps[-Ns] 1859 yield b 1860 while e is not None: 1861 yield e 1862 eventIndex -= 1 1863 e = self.getEvent(eventIndex)

1864 1869

1870 - def unsubscribe(self, receiver):

1871 self.watchers.remove(receiver)

1872 1873 ## Builder interface (methods called by the Builder which feeds us) 1874

1875 - def setSlavenames(self, names):

1876 self.slavenames = names

1877

1878 - def addEvent(self, text=[]):

1879 # this adds a duration event. When it is done, the user should call 1880 # e.finish(). They can also mangle it by modifying .text 1881 e = Event() 1882 e.started = util.now() 1883 e.text = text 1884 self.events.append(e) 1885 return e # they are free to mangle it further

1886

1887 - def addPointEvent(self, text=[]):

1888 # this adds a point event, one which occurs as a single atomic 1889 # instant of time. 1890 e = Event() 1891 e.started = util.now() 1892 e.finished = 0 1893 e.text = text 1894 self.events.append(e) 1895 return e # for consistency, but they really shouldn't touch it

1896

1897 - def setBigState(self, state):

1898 needToUpdate = state != self.currentBigState 1899 self.currentBigState = state 1900 if needToUpdate: 1901 self.publishState()

1902

1903 - def publishState(self, target=None):

1904 state = self.currentBigState 1905 1906 if target is not None: 1907 # unicast 1908 target.builderChangedState(self.name, state) 1909 return 1910 for w in self.watchers: 1911 try: 1912 w.builderChangedState(self.name, state) 1913 except: 1914 log.msg("Exception caught publishing state to %r" % w) 1915 log.err()

1916

1917 - def newBuild(self):

1918 """The Builder has decided to start a build, but the Build object is 1919 not yet ready to report status (it has not finished creating the 1920 Steps). Create a BuildStatus object that it can use.""" 1921 number = self.nextBuildNumber 1922 self.nextBuildNumber += 1 1923 # TODO: self.saveYourself(), to make sure we don't forget about the 1924 # build number we've just allocated. This is not quite as important 1925 # as it was before we switch to determineNextBuildNumber, but I think 1926 # it may still be useful to have the new build save itself. 1927 s = BuildStatus(self, number) 1928 s.waitUntilFinished().addCallback(self._buildFinished) 1929 return s

1930

1931 - def addBuildRequest(self, brstatus):

1932 self.pendingBuilds.append(brstatus) 1933 for w in self.watchers: 1934 w.requestSubmitted(brstatus)

1935

1936 - def removeBuildRequest(self, brstatus, cancelled=False):

1937 self.pendingBuilds.remove(brstatus) 1938 if cancelled: 1939 for w in self.watchers: 1940 w.requestCancelled(self, brstatus)

1941 1942 # buildStarted is called by our child BuildStatus instances

1943 - def buildStarted(self, s):

1944 """Now the BuildStatus object is ready to go (it knows all of its 1945 Steps, its ETA, etc), so it is safe to notify our watchers.""" 1946 1947 assert s.builder is self # paranoia 1948 assert s.number == self.nextBuildNumber - 1 1949 assert s not in self.currentBuilds 1950 self.currentBuilds.append(s) 1951 self.touchBuildCache(s) 1952 1953 # now that the BuildStatus is prepared to answer queries, we can 1954 # announce the new build to all our watchers 1955 1956 for w in self.watchers: # TODO: maybe do this later? callLater(0)? 1957 try: 1958 receiver = w.buildStarted(self.getName(), s) 1959 if receiver: 1960 if type(receiver) == type(()): 1961 s.subscribe(receiver[0], receiver[1]) 1962 else: 1963 s.subscribe(receiver) 1964 d = s.waitUntilFinished() 1965 d.addCallback(lambda s: s.unsubscribe(receiver)) 1966 except: 1967 log.msg("Exception caught notifying %r of buildStarted event" % w) 1968 log.err()

1969

1970 - def _buildFinished(self, s):

1971 assert s in self.currentBuilds 1972 s.saveYourself() 1973 self.currentBuilds.remove(s) 1974 1975 name = self.getName() 1976 results = s.getResults() 1977 for w in self.watchers: 1978 try: 1979 w.buildFinished(name, s, results) 1980 except: 1981 log.msg("Exception caught notifying %r of buildFinished event" % w) 1982 log.err() 1983 1984 self.prune() # conserve disk

1985 1986 1987 # waterfall display (history) 1988 1989 # I want some kind of build event that holds everything about the build: 1990 # why, what changes went into it, the results of the build, itemized 1991 # test results, etc. But, I do kind of need something to be inserted in 1992 # the event log first, because intermixing step events and the larger 1993 # build event is fraught with peril. Maybe an Event-like-thing that 1994 # doesn't have a file in it but does have links. Hmm, that's exactly 1995 # what it does now. The only difference would be that this event isn't 1996 # pushed to the clients. 1997 1998 # publish to clients

1999 - def sendLastBuildStatus(self, client):

2000 #client.newLastBuildStatus(self.lastBuildStatus) 2001 pass

2002 - def sendCurrentActivityBigToEveryone(self):

2003 for s in self.subscribers: 2004 self.sendCurrentActivityBig(s)

2005 - def sendCurrentActivityBig(self, client):

2006 state = self.currentBigState 2007 if state == "offline": 2008 client.currentlyOffline() 2009 elif state == "idle": 2010 client.currentlyIdle() 2011 elif state == "building": 2012 client.currentlyBuilding() 2013 else: 2014 log.msg("Hey, self.currentBigState is weird:", state)

2015 2016 2017 ## HTML display interface 2018

2019 - def getEventNumbered(self, num):

2020 # deal with dropped events, pruned events 2021 first = self.events[0].number 2022 if first + len(self.events)-1 != self.events[-1].number: 2023 log.msg(self, 2024 "lost an event somewhere: [0] is %d, [%d] is %d" % \ 2025 (self.events[0].number, 2026 len(self.events) - 1, 2027 self.events[-1].number)) 2028 for e in self.events: 2029 log.msg("e[%d]: " % e.number, e) 2030 return None 2031 offset = num - first 2032 log.msg(self, "offset", offset) 2033 try: 2034 return self.events[offset] 2035 except IndexError: 2036 return None

2037 2038 ## Persistence of Status

2039 - def loadYourOldEvents(self):

2040 if hasattr(self, "allEvents"): 2041 # first time, nothing to get from file. Note that this is only if 2042 # the Application gets .run() . If it gets .save()'ed, then the 2043 # .allEvents attribute goes away in the initial __getstate__ and 2044 # we try to load a non-existent file. 2045 return 2046 self.allEvents = self.loadFile("events", []) 2047 if self.allEvents: 2048 self.nextEventNumber = self.allEvents[-1].number + 1 2049 else: 2050 self.nextEventNumber = 0

2051 - def saveYourOldEvents(self):

2052 self.saveFile("events", self.allEvents)

2053 2054 ## clients 2055

2056 - def addClient(self, client):

2057 if client not in self.subscribers: 2058 self.subscribers.append(client) 2059 self.sendLastBuildStatus(client) 2060 self.sendCurrentActivityBig(client) 2061 client.newEvent(self.currentSmall)

2062 - def removeClient(self, client):

2063 if client in self.subscribers: 2064 self.subscribers.remove(client)

2065

2066 -class SlaveStatus:

2067 implements(interfaces.ISlaveStatus) 2068 2069 admin = None 2070 host = None 2071 access_uri = None 2072 version = None 2073 connected = False 2074 graceful_shutdown = False 2075

2076 - def __init__(self, name):

2077 self.name = name 2078 self._lastMessageReceived = 0 2079 self.runningBuilds = [] 2080 self.graceful_callbacks = []

2081

2082 - def getName(self):

2083 return self.name

2084 - def getAdmin(self):

2085 return self.admin

2086 - def getHost(self):

2087 return self.host

2088 - def getAccessURI(self):

2089 return self.access_uri

2090 - def getVersion(self):

2091 return self.version

2092 - def isConnected(self):

2093 return self.connected

2094 - def lastMessageReceived(self):

2095 return self._lastMessageReceived

2096 - def getRunningBuilds(self):

2097 return self.runningBuilds

2098

2099 - def setAdmin(self, admin):

2100 self.admin = admin

2101 - def setHost(self, host):

2102 self.host = host

2103 - def setAccessURI(self, access_uri):

2104 self.access_uri = access_uri

2105 - def setVersion(self, version):

2106 self.version = version

2107 - def setConnected(self, isConnected):

2108 self.connected = isConnected

2109 - def setLastMessageReceived(self, when):

2110 self._lastMessageReceived = when

2111

2112 - def buildStarted(self, build):

2113 self.runningBuilds.append(build)

2114 - def buildFinished(self, build):

2115 self.runningBuilds.remove(build)

2116

2117 - def getGraceful(self):

2118 """Return the graceful shutdown flag""" 2119 return self.graceful_shutdown

2120 - def setGraceful(self, graceful):

2121 """Set the graceful shutdown flag, and notify all the watchers""" 2122 self.graceful_shutdown = graceful 2123 for cb in self.graceful_callbacks: 2124 reactor.callLater(0, cb, graceful)

2125 - def addGracefulWatcher(self, watcher):

2126 """Add watcher to the list of watchers to be notified when the 2127 graceful shutdown flag is changed.""" 2128 if not watcher in self.graceful_callbacks: 2129 self.graceful_callbacks.append(watcher)

2130 - def removeGracefulWatcher(self, watcher):

2131 """Remove watcher from the list of watchers to be notified when the 2132 graceful shutdown flag is changed.""" 2133 if watcher in self.graceful_callbacks: 2134 self.graceful_callbacks.remove(watcher)

2135

2136 -class Status:

2137 """ 2138 I represent the status of the buildmaster. 2139 """ 2140 implements(interfaces.IStatus) 2141

2142 - def __init__(self, botmaster, basedir):

2143 """ 2144 @type botmaster: L{buildbot.master.BotMaster} 2145 @param botmaster: the Status object uses C{.botmaster} to get at 2146 both the L{buildbot.master.BuildMaster} (for 2147 various buildbot-wide parameters) and the 2148 actual Builders (to get at their L{BuilderStatus} 2149 objects). It is not allowed to change or influence 2150 anything through this reference. 2151 @type basedir: string 2152 @param basedir: this provides a base directory in which saved status 2153 information (changes.pck, saved Build status 2154 pickles) can be stored 2155 """ 2156 self.botmaster = botmaster 2157 self.basedir = basedir 2158 self.watchers = [] 2159 self.activeBuildSets = [] 2160 assert os.path.isdir(basedir) 2161 # compress logs bigger than 4k, a good default on linux 2162 self.logCompressionLimit = 4*1024 2163 self.logCompressionMethod = "bz2" 2164 # No default limit to the log size 2165 self.logMaxSize = None 2166 self.logMaxTailSize = None

2167 2168 2169 # methods called by our clients 2170

2171 - def getProjectName(self):

2172 return self.botmaster.parent.projectName

2173 - def getProjectURL(self):

2174 return self.botmaster.parent.projectURL

2175 - def getBuildbotURL(self):

2176 return self.botmaster.parent.buildbotURL

2177

2178 - def getURLForThing(self, thing):

2179 prefix = self.getBuildbotURL() 2180 if not prefix: 2181 return None 2182 if interfaces.IStatus.providedBy(thing): 2183 return prefix 2184 if interfaces.ISchedulerStatus.providedBy(thing): 2185 pass 2186 if interfaces.IBuilderStatus.providedBy(thing): 2187 builder = thing 2188 return prefix + "builders/%s" % ( 2189 urllib.quote(builder.getName(), safe=''), 2190 ) 2191 if interfaces.IBuildStatus.providedBy(thing): 2192 build = thing 2193 builder = build.getBuilder() 2194 return prefix + "builders/%s/builds/%d" % ( 2195 urllib.quote(builder.getName(), safe=''), 2196 build.getNumber()) 2197 if interfaces.IBuildStepStatus.providedBy(thing): 2198 step = thing 2199 build = step.getBuild() 2200 builder = build.getBuilder() 2201 return prefix + "builders/%s/builds/%d/steps/%s" % ( 2202 urllib.quote(builder.getName(), safe=''), 2203 build.getNumber(), 2204 urllib.quote(step.getName(), safe='')) 2205 # IBuildSetStatus 2206 # IBuildRequestStatus 2207 # ISlaveStatus 2208 2209 # IStatusEvent 2210 if interfaces.IStatusEvent.providedBy(thing): 2211 from buildbot.changes import changes 2212 # TODO: this is goofy, create IChange or something 2213 if isinstance(thing, changes.Change): 2214 change = thing 2215 return "%schanges/%d" % (prefix, change.number) 2216 2217 if interfaces.IStatusLog.providedBy(thing): 2218 log = thing 2219 step = log.getStep() 2220 build = step.getBuild() 2221 builder = build.getBuilder() 2222 2223 logs = step.getLogs() 2224 for i in range(len(logs)): 2225 if log is logs[i]: 2226 lognum = i 2227 break 2228 else: 2229 return None 2230 return prefix + "builders/%s/builds/%d/steps/%s/logs/%s" % ( 2231 urllib.quote(builder.getName(), safe=''), 2232 build.getNumber(), 2233 urllib.quote(step.getName(), safe=''), 2234 urllib.quote(log.getName()))

2235

2236 - def getChangeSources(self):

2237 return list(self.botmaster.parent.change_svc)

2238

2239 - def getChange(self, number):

2240 return self.botmaster.parent.change_svc.getChangeNumbered(number)

2241

2242 - def getSchedulers(self):

2243 return self.botmaster.parent.allSchedulers()

2244

2245 - def getBuilderNames(self, categories=None):

2246 if categories == None: 2247 return self.botmaster.builderNames[:] # don't let them break it 2248 2249 l = [] 2250 # respect addition order 2251 for name in self.botmaster.builderNames: 2252 builder = self.botmaster.builders[name] 2253 if builder.builder_status.category in categories: 2254 l.append(name) 2255 return l

2256

2257 - def getBuilder(self, name):

2258 """ 2259 @rtype: L{BuilderStatus} 2260 """ 2261 return self.botmaster.builders[name].builder_status

2262

2263 - def getSlaveNames(self):

2264 return self.botmaster.slaves.keys()

2265

2266 - def getSlave(self, slavename):

2267 return self.botmaster.slaves[slavename].slave_status

2268

2269 - def getBuildSets(self):

2270 return self.activeBuildSets[:]

2271

2272 - def generateFinishedBuilds(self, builders=[], branches=[], 2273 num_builds=None, finished_before=None, 2274 max_search=200):

2275 2276 def want_builder(bn): 2277 if builders: 2278 return bn in builders 2279 return True

2280 builder_names = [bn 2281 for bn in self.getBuilderNames() 2282 if want_builder(bn)] 2283 2284 # 'sources' is a list of generators, one for each Builder we're 2285 # using. When the generator is exhausted, it is replaced in this list 2286 # with None. 2287 sources = [] 2288 for bn in builder_names: 2289 b = self.getBuilder(bn) 2290 g = b.generateFinishedBuilds(branches, 2291 finished_before=finished_before, 2292 max_search=max_search) 2293 sources.append(g) 2294 2295 # next_build the next build from each source 2296 next_build = [None] * len(sources) 2297 2298 def refill(): 2299 for i,g in enumerate(sources): 2300 if next_build[i]: 2301 # already filled 2302 continue 2303 if not g: 2304 # already exhausted 2305 continue 2306 try: 2307 next_build[i] = g.next() 2308 except StopIteration: 2309 next_build[i] = None 2310 sources[i] = None

2311 2312 got = 0 2313 while True: 2314 refill() 2315 # find the latest build among all the candidates 2316 candidates = [(i, b, b.getTimes()[1]) 2317 for i,b in enumerate(next_build) 2318 if b is not None] 2319 candidates.sort(lambda x,y: cmp(x[2], y[2])) 2320 if not candidates: 2321 return 2322 2323 # and remove it from the list 2324 i, build, finshed_time = candidates[-1] 2325 next_build[i] = None 2326 got += 1 2327 yield build 2328 if num_builds is not None: 2329 if got >= num_builds: 2330 return 2331

2336 - def unsubscribe(self, target):

2337 self.watchers.remove(target)

2338 2339 2340 # methods called by upstream objects 2341

2342 - def announceNewBuilder(self, target, name, builder_status):

2343 t = target.builderAdded(name, builder_status) 2344 if t: 2345 builder_status.subscribe(t)

2346

2347 - def builderAdded(self, name, basedir, category=None):

2348 """ 2349 @rtype: L{BuilderStatus} 2350 """ 2351 filename = os.path.join(self.basedir, basedir, "builder") 2352 log.msg("trying to load status pickle from %s" % filename) 2353 builder_status = None 2354 try: 2355 builder_status = load(open(filename, "rb")) 2356 styles.doUpgrade() 2357 except IOError: 2358 log.msg("no saved status pickle, creating a new one") 2359 except: 2360 log.msg("error while loading status pickle, creating a new one") 2361 log.msg("error follows:") 2362 log.err() 2363 if not builder_status: 2364 builder_status = BuilderStatus(name, category) 2365 builder_status.addPointEvent(["builder", "created"]) 2366 log.msg("added builder %s in category %s" % (name, category)) 2367 # an unpickled object might not have category set from before, 2368 # so set it here to make sure 2369 builder_status.category = category 2370 builder_status.basedir = os.path.join(self.basedir, basedir) 2371 builder_status.name = name # it might have been updated 2372 builder_status.status = self 2373 2374 if not os.path.isdir(builder_status.basedir): 2375 os.makedirs(builder_status.basedir) 2376 builder_status.determineNextBuildNumber() 2377 2378 builder_status.setBigState("offline") 2379 builder_status.setLogCompressionLimit(self.logCompressionLimit) 2380 builder_status.setLogCompressionMethod(self.logCompressionMethod) 2381 builder_status.setLogMaxSize(self.logMaxSize) 2382 builder_status.setLogMaxTailSize(self.logMaxTailSize) 2383 2384 for t in self.watchers: 2385 self.announceNewBuilder(t, name, builder_status) 2386 2387 return builder_status

2388

2389 - def builderRemoved(self, name):

2390 for t in self.watchers: 2391 t.builderRemoved(name)

2392

2393 - def buildsetSubmitted(self, bss):

2394 self.activeBuildSets.append(bss) 2395 bss.waitUntilFinished().addCallback(self.activeBuildSets.remove) 2396 for t in self.watchers: 2397 t.buildsetSubmitted(bss)

2398

Source Code for Module buildbot.status.builder