buildbot.status.builder

1 # -*- test-case-name: buildbot.test.test_status -*- 2 3 from zope.interface import implements 4 from twisted.python import log, runtime 5 from twisted.persisted import styles 6 from twisted.internet import reactor, defer, threads 7 from twisted.protocols import basic 8 from buildbot.process.properties import Properties 9 from buildbot.util import collections 10 from buildbot.util.eventual import eventually 11 12 import weakref 13 import os, shutil, re, urllib, itertools 14 import gc 15 import time 16 from cPickle import load, dump 17 from cStringIO import StringIO 18 from bz2 import BZ2File 19 from gzip import GzipFile 20 21 # sibling imports 22 from buildbot import interfaces, util, sourcestamp 23 24 SUCCESS, WARNINGS, FAILURE, SKIPPED, EXCEPTION, RETRY = range(6) 25 Results = ["success", "warnings", "failure", "skipped", "exception", "retry"]

26 27 -def worst_status(a, b):

28 # SUCCESS > SKIPPED > WARNINGS > FAILURE > EXCEPTION > RETRY 29 # Retry needs to be considered the worst so that conusmers don't have to 30 # worry about other failures undermining the RETRY. 31 for s in (RETRY, EXCEPTION, FAILURE, WARNINGS, SKIPPED, SUCCESS): 32 if s in (a, b): 33 return s

34 35 # build processes call the following methods: 36 # 37 # setDefaults 38 # 39 # currentlyBuilding 40 # currentlyIdle 41 # currentlyInterlocked 42 # currentlyOffline 43 # currentlyWaiting 44 # 45 # setCurrentActivity 46 # updateCurrentActivity 47 # addFileToCurrentActivity 48 # finishCurrentActivity 49 # 50 # startBuild 51 # finishBuild 52 53 STDOUT = interfaces.LOG_CHANNEL_STDOUT 54 STDERR = interfaces.LOG_CHANNEL_STDERR 55 HEADER = interfaces.LOG_CHANNEL_HEADER 56 ChunkTypes = ["stdout", "stderr", "header"]

57 58 -class LogFileScanner(basic.NetstringReceiver):

59 - def __init__(self, chunk_cb, channels=[]):

60 self.chunk_cb = chunk_cb 61 self.channels = channels

62

63 - def stringReceived(self, line):

64 channel = int(line[0]) 65 if not self.channels or (channel in self.channels): 66 self.chunk_cb((channel, line[1:]))

67

68 -class LogFileProducer:

69 """What's the plan? 70 71 the LogFile has just one FD, used for both reading and writing. 72 Each time you add an entry, fd.seek to the end and then write. 73 74 Each reader (i.e. Producer) keeps track of their own offset. The reader 75 starts by seeking to the start of the logfile, and reading forwards. 76 Between each hunk of file they yield chunks, so they must remember their 77 offset before yielding and re-seek back to that offset before reading 78 more data. When their read() returns EOF, they're finished with the first 79 phase of the reading (everything that's already been written to disk). 80 81 After EOF, the remaining data is entirely in the current entries list. 82 These entries are all of the same channel, so we can do one "".join and 83 obtain a single chunk to be sent to the listener. But since that involves 84 a yield, and more data might arrive after we give up control, we have to 85 subscribe them before yielding. We can't subscribe them any earlier, 86 otherwise they'd get data out of order. 87 88 We're using a generator in the first place so that the listener can 89 throttle us, which means they're pulling. But the subscription means 90 we're pushing. Really we're a Producer. In the first phase we can be 91 either a PullProducer or a PushProducer. In the second phase we're only a 92 PushProducer. 93 94 So the client gives a LogFileConsumer to File.subscribeConsumer . This 95 Consumer must have registerProducer(), unregisterProducer(), and 96 writeChunk(), and is just like a regular twisted.interfaces.IConsumer, 97 except that writeChunk() takes chunks (tuples of (channel,text)) instead 98 of the normal write() which takes just text. The LogFileConsumer is 99 allowed to call stopProducing, pauseProducing, and resumeProducing on the 100 producer instance it is given. """ 101 102 paused = False 103 subscribed = False 104 BUFFERSIZE = 2048 105

106 - def __init__(self, logfile, consumer):

107 self.logfile = logfile 108 self.consumer = consumer 109 self.chunkGenerator = self.getChunks() 110 consumer.registerProducer(self, True)

111

112 - def getChunks(self):

113 f = self.logfile.getFile() 114 offset = 0 115 chunks = [] 116 p = LogFileScanner(chunks.append) 117 f.seek(offset) 118 data = f.read(self.BUFFERSIZE) 119 offset = f.tell() 120 while data: 121 p.dataReceived(data) 122 while chunks: 123 c = chunks.pop(0) 124 yield c 125 f.seek(offset) 126 data = f.read(self.BUFFERSIZE) 127 offset = f.tell() 128 del f 129 130 # now subscribe them to receive new entries 131 self.subscribed = True 132 self.logfile.watchers.append(self) 133 d = self.logfile.waitUntilFinished() 134 135 # then give them the not-yet-merged data 136 if self.logfile.runEntries: 137 channel = self.logfile.runEntries[0][0] 138 text = "".join([c[1] for c in self.logfile.runEntries]) 139 yield (channel, text) 140 141 # now we've caught up to the present. Anything further will come from 142 # the logfile subscription. We add the callback *after* yielding the 143 # data from runEntries, because the logfile might have finished 144 # during the yield. 145 d.addCallback(self.logfileFinished)

146

147 - def stopProducing(self):

148 # TODO: should we still call consumer.finish? probably not. 149 self.paused = True 150 self.consumer = None 151 self.done()

152

153 - def done(self):

154 if self.chunkGenerator: 155 self.chunkGenerator = None # stop making chunks 156 if self.subscribed: 157 self.logfile.watchers.remove(self) 158 self.subscribed = False

159

160 - def pauseProducing(self):

161 self.paused = True

162

163 - def resumeProducing(self):

164 # Twisted-1.3.0 has a bug which causes hangs when resumeProducing 165 # calls transport.write (there is a recursive loop, fixed in 2.0 in 166 # t.i.abstract.FileDescriptor.doWrite by setting the producerPaused 167 # flag *before* calling resumeProducing). To work around this, we 168 # just put off the real resumeProducing for a moment. This probably 169 # has a performance hit, but I'm going to assume that the log files 170 # are not retrieved frequently enough for it to be an issue. 171 172 eventually(self._resumeProducing)

173

174 - def _resumeProducing(self):

175 self.paused = False 176 if not self.chunkGenerator: 177 return 178 try: 179 while not self.paused: 180 chunk = self.chunkGenerator.next() 181 self.consumer.writeChunk(chunk) 182 # we exit this when the consumer says to stop, or we run out 183 # of chunks 184 except StopIteration: 185 # if the generator finished, it will have done releaseFile 186 self.chunkGenerator = None

187 # now everything goes through the subscription, and they don't get to 188 # pause anymore 189

190 - def logChunk(self, build, step, logfile, channel, chunk):

191 if self.consumer: 192 self.consumer.writeChunk((channel, chunk))

193

194 - def logfileFinished(self, logfile):

195 self.done() 196 if self.consumer: 197 self.consumer.unregisterProducer() 198 self.consumer.finish() 199 self.consumer = None

200

201 -def _tryremove(filename, timeout, retries):

202 """Try to remove a file, and if failed, try again in timeout. 203 Increases the timeout by a factor of 4, and only keeps trying for 204 another retries-amount of times. 205 206 """ 207 try: 208 os.unlink(filename) 209 except OSError: 210 if retries > 0: 211 reactor.callLater(timeout, _tryremove, filename, timeout * 4, 212 retries - 1) 213 else: 214 log.msg("giving up on removing %s after over %d seconds" % 215 (filename, timeout))

216

217 -class LogFile:

218 """A LogFile keeps all of its contents on disk, in a non-pickle format to 219 which new entries can easily be appended. The file on disk has a name 220 like 12-log-compile-output, under the Builder's directory. The actual 221 filename is generated (before the LogFile is created) by 222 L{BuildStatus.generateLogfileName}. 223 224 Old LogFile pickles (which kept their contents in .entries) must be 225 upgraded. The L{BuilderStatus} is responsible for doing this, when it 226 loads the L{BuildStatus} into memory. The Build pickle is not modified, 227 so users who go from 0.6.5 back to 0.6.4 don't have to lose their 228 logs.""" 229 230 implements(interfaces.IStatusLog, interfaces.ILogFile) 231 232 finished = False 233 length = 0 234 nonHeaderLength = 0 235 tailLength = 0 236 chunkSize = 10*1000 237 runLength = 0 238 # No max size by default 239 logMaxSize = None 240 # Don't keep a tail buffer by default 241 logMaxTailSize = None 242 maxLengthExceeded = False 243 runEntries = [] # provided so old pickled builds will getChunks() ok 244 entries = None 245 BUFFERSIZE = 2048 246 filename = None # relative to the Builder's basedir 247 openfile = None 248 compressMethod = "bz2" 249

250 - def __init__(self, parent, name, logfilename):

251 """ 252 @type parent: L{BuildStepStatus} 253 @param parent: the Step that this log is a part of 254 @type name: string 255 @param name: the name of this log, typically 'output' 256 @type logfilename: string 257 @param logfilename: the Builder-relative pathname for the saved entries 258 """ 259 self.step = parent 260 self.name = name 261 self.filename = logfilename 262 fn = self.getFilename() 263 if os.path.exists(fn): 264 # the buildmaster was probably stopped abruptly, before the 265 # BuilderStatus could be saved, so BuilderStatus.nextBuildNumber 266 # is out of date, and we're overlapping with earlier builds now. 267 # Warn about it, but then overwrite the old pickle file 268 log.msg("Warning: Overwriting old serialized Build at %s" % fn) 269 dirname = os.path.dirname(fn) 270 if not os.path.exists(dirname): 271 os.makedirs(dirname) 272 self.openfile = open(fn, "w+") 273 self.runEntries = [] 274 self.watchers = [] 275 self.finishedWatchers = [] 276 self.tailBuffer = []

277

278 - def getFilename(self):

279 return os.path.join(self.step.build.builder.basedir, self.filename)

280

281 - def hasContents(self):

282 return os.path.exists(self.getFilename() + '.bz2') or \ 283 os.path.exists(self.getFilename() + '.gz') or \ 284 os.path.exists(self.getFilename())

285

286 - def getName(self):

287 return self.name

288

289 - def getStep(self):

290 return self.step

291

292 - def isFinished(self):

293 return self.finished

294 - def waitUntilFinished(self):

295 if self.finished: 296 d = defer.succeed(self) 297 else: 298 d = defer.Deferred() 299 self.finishedWatchers.append(d) 300 return d

301

302 - def getFile(self):

303 if self.openfile: 304 # this is the filehandle we're using to write to the log, so 305 # don't close it! 306 return self.openfile 307 # otherwise they get their own read-only handle 308 # try a compressed log first 309 try: 310 return BZ2File(self.getFilename() + ".bz2", "r") 311 except IOError: 312 pass 313 try: 314 return GzipFile(self.getFilename() + ".gz", "r") 315 except IOError: 316 pass 317 return open(self.getFilename(), "r")

318

319 - def getText(self):

320 # this produces one ginormous string 321 return "".join(self.getChunks([STDOUT, STDERR], onlyText=True))

322

323 - def getTextWithHeaders(self):

324 return "".join(self.getChunks(onlyText=True))

325

326 - def getChunks(self, channels=[], onlyText=False):

327 # generate chunks for everything that was logged at the time we were 328 # first called, so remember how long the file was when we started. 329 # Don't read beyond that point. The current contents of 330 # self.runEntries will follow. 331 332 # this returns an iterator, which means arbitrary things could happen 333 # while we're yielding. This will faithfully deliver the log as it 334 # existed when it was started, and not return anything after that 335 # point. To use this in subscribe(catchup=True) without missing any 336 # data, you must insure that nothing will be added to the log during 337 # yield() calls. 338 339 f = self.getFile() 340 if not self.finished: 341 offset = 0 342 f.seek(0, 2) 343 remaining = f.tell() 344 else: 345 offset = 0 346 remaining = None 347 348 leftover = None 349 if self.runEntries and (not channels or 350 (self.runEntries[0][0] in channels)): 351 leftover = (self.runEntries[0][0], 352 "".join([c[1] for c in self.runEntries])) 353 354 # freeze the state of the LogFile by passing a lot of parameters into 355 # a generator 356 return self._generateChunks(f, offset, remaining, leftover, 357 channels, onlyText)

358

359 - def _generateChunks(self, f, offset, remaining, leftover, 360 channels, onlyText):

361 chunks = [] 362 p = LogFileScanner(chunks.append, channels) 363 f.seek(offset) 364 if remaining is not None: 365 data = f.read(min(remaining, self.BUFFERSIZE)) 366 remaining -= len(data) 367 else: 368 data = f.read(self.BUFFERSIZE) 369 370 offset = f.tell() 371 while data: 372 p.dataReceived(data) 373 while chunks: 374 channel, text = chunks.pop(0) 375 if onlyText: 376 yield text 377 else: 378 yield (channel, text) 379 f.seek(offset) 380 if remaining is not None: 381 data = f.read(min(remaining, self.BUFFERSIZE)) 382 remaining -= len(data) 383 else: 384 data = f.read(self.BUFFERSIZE) 385 offset = f.tell() 386 del f 387 388 if leftover: 389 if onlyText: 390 yield leftover[1] 391 else: 392 yield leftover

393

394 - def readlines(self, channel=STDOUT):

395 """Return an iterator that produces newline-terminated lines, 396 excluding header chunks.""" 397 # TODO: make this memory-efficient, by turning it into a generator 398 # that retrieves chunks as necessary, like a pull-driven version of 399 # twisted.protocols.basic.LineReceiver 400 alltext = "".join(self.getChunks([channel], onlyText=True)) 401 io = StringIO(alltext) 402 return io.readlines()

403 413

414 - def unsubscribe(self, receiver):

415 if receiver in self.watchers: 416 self.watchers.remove(receiver)

417

418 - def subscribeConsumer(self, consumer):

419 p = LogFileProducer(self, consumer) 420 p.resumeProducing()

421 422 # interface used by the build steps to add things to the log 423

424 - def merge(self):

425 # merge all .runEntries (which are all of the same type) into a 426 # single chunk for .entries 427 if not self.runEntries: 428 return 429 channel = self.runEntries[0][0] 430 text = "".join([c[1] for c in self.runEntries]) 431 assert channel < 10 432 f = self.openfile 433 f.seek(0, 2) 434 offset = 0 435 while offset < len(text): 436 size = min(len(text)-offset, self.chunkSize) 437 f.write("%d:%d" % (1 + size, channel)) 438 f.write(text[offset:offset+size]) 439 f.write(",") 440 offset += size 441 self.runEntries = [] 442 self.runLength = 0

443

444 - def addEntry(self, channel, text):

445 assert not self.finished 446 447 if isinstance(text, unicode): 448 text = text.encode('utf-8') 449 if channel != HEADER: 450 # Truncate the log if it's more than logMaxSize bytes 451 if self.logMaxSize and self.nonHeaderLength > self.logMaxSize: 452 # Add a message about what's going on 453 if not self.maxLengthExceeded: 454 msg = "\nOutput exceeded %i bytes, remaining output has been truncated\n" % self.logMaxSize 455 self.addEntry(HEADER, msg) 456 self.merge() 457 self.maxLengthExceeded = True 458 459 if self.logMaxTailSize: 460 # Update the tail buffer 461 self.tailBuffer.append((channel, text)) 462 self.tailLength += len(text) 463 while self.tailLength > self.logMaxTailSize: 464 # Drop some stuff off the beginning of the buffer 465 c,t = self.tailBuffer.pop(0) 466 n = len(t) 467 self.tailLength -= n 468 assert self.tailLength >= 0 469 return 470 471 self.nonHeaderLength += len(text) 472 473 # we only add to .runEntries here. merge() is responsible for adding 474 # merged chunks to .entries 475 if self.runEntries and channel != self.runEntries[0][0]: 476 self.merge() 477 self.runEntries.append((channel, text)) 478 self.runLength += len(text) 479 if self.runLength >= self.chunkSize: 480 self.merge() 481 482 for w in self.watchers: 483 w.logChunk(self.step.build, self.step, self, channel, text) 484 self.length += len(text)

485

486 - def addStdout(self, text):

487 self.addEntry(STDOUT, text)

488 - def addStderr(self, text):

489 self.addEntry(STDERR, text)

490 - def addHeader(self, text):

491 self.addEntry(HEADER, text)

492

493 - def finish(self):

494 if self.tailBuffer: 495 msg = "\nFinal %i bytes follow below:\n" % self.tailLength 496 tmp = self.runEntries 497 self.runEntries = [(HEADER, msg)] 498 self.merge() 499 self.runEntries = self.tailBuffer 500 self.merge() 501 self.runEntries = tmp 502 self.merge() 503 self.tailBuffer = [] 504 else: 505 self.merge() 506 507 if self.openfile: 508 # we don't do an explicit close, because there might be readers 509 # shareing the filehandle. As soon as they stop reading, the 510 # filehandle will be released and automatically closed. 511 self.openfile.flush() 512 del self.openfile 513 self.finished = True 514 watchers = self.finishedWatchers 515 self.finishedWatchers = [] 516 for w in watchers: 517 w.callback(self) 518 self.watchers = []

519 520

521 - def compressLog(self):

522 # bail out if there's no compression support 523 if self.compressMethod == "bz2": 524 compressed = self.getFilename() + ".bz2.tmp" 525 elif self.compressMethod == "gz": 526 compressed = self.getFilename() + ".gz.tmp" 527 d = threads.deferToThread(self._compressLog, compressed) 528 d.addCallback(self._renameCompressedLog, compressed) 529 d.addErrback(self._cleanupFailedCompress, compressed) 530 return d

531

532 - def _compressLog(self, compressed):

533 infile = self.getFile() 534 if self.compressMethod == "bz2": 535 cf = BZ2File(compressed, 'w') 536 elif self.compressMethod == "gz": 537 cf = GzipFile(compressed, 'w') 538 bufsize = 1024*1024 539 while True: 540 buf = infile.read(bufsize) 541 cf.write(buf) 542 if len(buf) < bufsize: 543 break 544 cf.close()

545 - def _renameCompressedLog(self, rv, compressed):

546 if self.compressMethod == "bz2": 547 filename = self.getFilename() + '.bz2' 548 else: 549 filename = self.getFilename() + '.gz' 550 if runtime.platformType == 'win32': 551 # windows cannot rename a file on top of an existing one, so 552 # fall back to delete-first. There are ways this can fail and 553 # lose the builder's history, so we avoid using it in the 554 # general (non-windows) case 555 if os.path.exists(filename): 556 os.unlink(filename) 557 os.rename(compressed, filename) 558 _tryremove(self.getFilename(), 1, 5)

559 - def _cleanupFailedCompress(self, failure, compressed):

560 log.msg("failed to compress %s" % self.getFilename()) 561 if os.path.exists(compressed): 562 _tryremove(compressed, 1, 5) 563 failure.trap() # reraise the failure

564 565 # persistence stuff

566 - def __getstate__(self):

567 d = self.__dict__.copy() 568 del d['step'] # filled in upon unpickling 569 del d['watchers'] 570 del d['finishedWatchers'] 571 d['entries'] = [] # let 0.6.4 tolerate the saved log. TODO: really? 572 if d.has_key('finished'): 573 del d['finished'] 574 if d.has_key('openfile'): 575 del d['openfile'] 576 return d

577

578 - def __setstate__(self, d):

579 self.__dict__ = d 580 self.watchers = [] # probably not necessary 581 self.finishedWatchers = [] # same 582 # self.step must be filled in by our parent 583 self.finished = True

584

585 - def upgrade(self, logfilename):

586 """Save our .entries to a new-style offline log file (if necessary), 587 and modify our in-memory representation to use it. The original 588 pickled LogFile (inside the pickled Build) won't be modified.""" 589 self.filename = logfilename 590 if not os.path.exists(self.getFilename()): 591 self.openfile = open(self.getFilename(), "w") 592 self.finished = False 593 for channel,text in self.entries: 594 self.addEntry(channel, text) 595 self.finish() # releases self.openfile, which will be closed 596 del self.entries

597

598 -class HTMLLogFile:

599 implements(interfaces.IStatusLog) 600 601 filename = None 602

603 - def __init__(self, parent, name, logfilename, html):

604 self.step = parent 605 self.name = name 606 self.filename = logfilename 607 self.html = html

608

609 - def getName(self):

610 return self.name # set in BuildStepStatus.addLog

611 - def getStep(self):

612 return self.step

613

614 - def isFinished(self):

615 return True

616 - def waitUntilFinished(self):

617 return defer.succeed(self)

618

619 - def hasContents(self):

620 return True

621 - def getText(self):

622 return self.html # looks kinda like text

623 - def getTextWithHeaders(self):

624 return self.html

625 - def getChunks(self):

626 return [(STDERR, self.html)]

627

630 - def unsubscribe(self, receiver):

631 pass

632

633 - def finish(self):

634 pass

635

636 - def __getstate__(self):

637 d = self.__dict__.copy() 638 del d['step'] 639 return d

640

641 - def upgrade(self, logfilename):

642 pass

643

644 645 -class Event:

646 implements(interfaces.IStatusEvent) 647 648 started = None 649 finished = None 650 text = [] 651 652 # IStatusEvent methods

653 - def getTimes(self):

654 return (self.started, self.finished)

655 - def getText(self):

656 return self.text

657 - def getLogs(self):

658 return []

659

660 - def finish(self):

661 self.finished = util.now()

662

663 -class TestResult:

664 implements(interfaces.ITestResult) 665

666 - def __init__(self, name, results, text, logs):

667 assert isinstance(name, tuple) 668 self.name = name 669 self.results = results 670 self.text = text 671 self.logs = logs

672

673 - def getName(self):

674 return self.name

675

676 - def getResults(self):

677 return self.results

678

679 - def getText(self):

680 return self.text

681

682 - def getLogs(self):

683 return self.logs

684

685 686 -class BuildSetStatus:

687 implements(interfaces.IBuildSetStatus) 688

689 - def __init__(self, bsid, status, db):

690 self.id = bsid 691 self.status = status 692 self.db = db

693

694 - def _get_info(self):

695 return self.db.get_buildset_info(self.id)

696 697 # methods for our clients 698

699 - def getSourceStamp(self):

700 (external_idstring, reason, ssid, complete, results) = self._get_info() 701 return self.db.getSourceStampNumberedNow(ssid)

702

703 - def getReason(self):

704 (external_idstring, reason, ssid, complete, results) = self._get_info() 705 return reason

706 - def getResults(self):

707 (external_idstring, reason, ssid, complete, results) = self._get_info() 708 return results

709 - def getID(self):

710 (external_idstring, reason, ssid, complete, results) = self._get_info() 711 return external_idstring

712

713 - def getBuilderNamesAndBuildRequests(self):

714 brs = {} 715 brids = self.db.get_buildrequestids_for_buildset(self.id) 716 for (buildername, brid) in brids.items(): 717 brs[buildername] = BuildRequestStatus(brid, self.status, self.db) 718 return brs

719

720 - def getBuilderNames(self):

721 brs = self.db.get_buildrequestids_for_buildset(self.id) 722 return sorted(brs.keys())

723

724 - def getBuildRequests(self):

725 brs = self.db.get_buildrequestids_for_buildset(self.id) 726 return [BuildRequestStatus(brid, self.status, self.db) 727 for brid in brs.values()]

728

729 - def isFinished(self):

730 (external_idstring, reason, ssid, complete, results) = self._get_info() 731 return complete

732

733 - def waitUntilSuccess(self):

734 return self.status._buildset_waitUntilSuccess(self.id)

735 - def waitUntilFinished(self):

736 return self.status._buildset_waitUntilFinished(self.id)

737

738 -class BuildRequestStatus:

739 implements(interfaces.IBuildRequestStatus) 740

741 - def __init__(self, brid, status, db):

742 self.brid = brid 743 self.status = status 744 self.db = db

745

746 - def buildStarted(self, build):

747 self.status._buildrequest_buildStarted(build.status) 748 self.builds.append(build.status)

749 750 # methods called by our clients

751 - def getSourceStamp(self):

752 br = self.db.getBuildRequestWithNumber(self.brid) 753 return br.source

754 - def getBuilderName(self):

755 br = self.db.getBuildRequestWithNumber(self.brid) 756 return br.builderName

757 - def getBuilds(self):

758 builder = self.status.getBuilder(self.getBuilderName()) 759 builds = [] 760 buildnums = sorted(self.db.get_buildnums_for_brid(self.brid)) 761 for buildnum in buildnums: 762 bs = builder.getBuild(buildnum) 763 if bs: 764 builds.append(bs) 765 return builds

766

772 - def unsubscribe(self, observer):

773 self.status._buildrequest_unsubscribe(self.brid, observer)

774

775 - def getSubmitTime(self):

776 br = self.db.getBuildRequestWithNumber(self.brid) 777 return br.submittedAt

778

779 - def asDict(self):

780 result = {} 781 # Constant 782 result['source'] = self.getSourceStamp().asDict() 783 result['builderName'] = self.getBuilderName() 784 result['submittedAt'] = self.getSubmitTime() 785 786 # Transient 787 result['builds'] = [build.asDict() for build in self.getBuilds()] 788 return result

789

790 791 -class BuildStepStatus(styles.Versioned):

792 """ 793 I represent a collection of output status for a 794 L{buildbot.process.step.BuildStep}. 795 796 Statistics contain any information gleaned from a step that is 797 not in the form of a logfile. As an example, steps that run 798 tests might gather statistics about the number of passed, failed, 799 or skipped tests. 800 801 @type progress: L{buildbot.status.progress.StepProgress} 802 @cvar progress: tracks ETA for the step 803 @type text: list of strings 804 @cvar text: list of short texts that describe the command and its status 805 @type text2: list of strings 806 @cvar text2: list of short texts added to the overall build description 807 @type logs: dict of string -> L{buildbot.status.builder.LogFile} 808 @ivar logs: logs of steps 809 @type statistics: dict 810 @ivar statistics: results from running this step 811 """ 812 # note that these are created when the Build is set up, before each 813 # corresponding BuildStep has started. 814 implements(interfaces.IBuildStepStatus, interfaces.IStatusEvent) 815 persistenceVersion = 3 816 817 started = None 818 finished = None 819 progress = None 820 text = [] 821 results = (None, []) 822 text2 = [] 823 watchers = [] 824 updates = {} 825 finishedWatchers = [] 826 statistics = {} 827 step_number = None 828

829 - def __init__(self, parent, step_number):

830 assert interfaces.IBuildStatus(parent) 831 self.build = parent 832 self.step_number = step_number 833 self.logs = [] 834 self.urls = {} 835 self.watchers = [] 836 self.updates = {} 837 self.finishedWatchers = [] 838 self.statistics = {} 839 self.skipped = False 840 841 self.waitingForLocks = False

842

843 - def getName(self):

844 """Returns a short string with the name of this step. This string 845 may have spaces in it.""" 846 return self.name

847

848 - def getBuild(self):

849 return self.build

850

851 - def getTimes(self):

852 return (self.started, self.finished)

853

854 - def getExpectations(self):

855 """Returns a list of tuples (name, current, target).""" 856 if not self.progress: 857 return [] 858 ret = [] 859 metrics = self.progress.progress.keys() 860 metrics.sort() 861 for m in metrics: 862 t = (m, self.progress.progress[m], self.progress.expectations[m]) 863 ret.append(t) 864 return ret

865

866 - def getLogs(self):

867 return self.logs

868

869 - def getURLs(self):

870 return self.urls.copy()

871

872 - def isStarted(self):

873 return (self.started is not None)

874

875 - def isSkipped(self):

876 return self.skipped

877

878 - def isFinished(self):

879 return (self.finished is not None)

880

881 - def waitUntilFinished(self):

882 if self.finished: 883 d = defer.succeed(self) 884 else: 885 d = defer.Deferred() 886 self.finishedWatchers.append(d) 887 return d

888 889 # while the step is running, the following methods make sense. 890 # Afterwards they return None 891

892 - def getETA(self):

893 if self.started is None: 894 return None # not started yet 895 if self.finished is not None: 896 return None # already finished 897 if not self.progress: 898 return None # no way to predict 899 return self.progress.remaining()

900 901 # Once you know the step has finished, the following methods are legal. 902 # Before this step has finished, they all return None. 903

904 - def getText(self):

905 """Returns a list of strings which describe the step. These are 906 intended to be displayed in a narrow column. If more space is 907 available, the caller should join them together with spaces before 908 presenting them to the user.""" 909 return self.text

910

911 - def getResults(self):

912 """Return a tuple describing the results of the step. 913 'result' is one of the constants in L{buildbot.status.builder}: 914 SUCCESS, WARNINGS, FAILURE, or SKIPPED. 915 'strings' is an optional list of strings that the step wants to 916 append to the overall build's results. These strings are usually 917 more terse than the ones returned by getText(): in particular, 918 successful Steps do not usually contribute any text to the 919 overall build. 920 921 @rtype: tuple of int, list of strings 922 @returns: (result, strings) 923 """ 924 return (self.results, self.text2)

925

926 - def hasStatistic(self, name):

927 """Return true if this step has a value for the given statistic. 928 """ 929 return self.statistics.has_key(name)

930

931 - def getStatistic(self, name, default=None):

932 """Return the given statistic, if present 933 """ 934 return self.statistics.get(name, default)

935 936 # subscription interface 937 943

944 - def sendETAUpdate(self, receiver, updateInterval):

945 self.updates[receiver] = None 946 # they might unsubscribe during stepETAUpdate 947 receiver.stepETAUpdate(self.build, self, 948 self.getETA(), self.getExpectations()) 949 if receiver in self.watchers: 950 self.updates[receiver] = reactor.callLater(updateInterval, 951 self.sendETAUpdate, 952 receiver, 953 updateInterval)

954

955 - def unsubscribe(self, receiver):

956 if receiver in self.watchers: 957 self.watchers.remove(receiver) 958 if receiver in self.updates: 959 if self.updates[receiver] is not None: 960 self.updates[receiver].cancel() 961 del self.updates[receiver]

962 963 964 # methods to be invoked by the BuildStep 965

966 - def setName(self, stepname):

967 self.name = stepname

968

969 - def setColor(self, color):

970 log.msg("BuildStepStatus.setColor is no longer supported -- ignoring color %s" % (color,))

971

972 - def setProgress(self, stepprogress):

973 self.progress = stepprogress

974

975 - def stepStarted(self):

976 self.started = util.now() 977 if self.build: 978 self.build.stepStarted(self)

979

980 - def addLog(self, name):

981 assert self.started # addLog before stepStarted won't notify watchers 982 logfilename = self.build.generateLogfileName(self.name, name) 983 log = LogFile(self, name, logfilename) 984 log.logMaxSize = self.build.builder.logMaxSize 985 log.logMaxTailSize = self.build.builder.logMaxTailSize 986 log.compressMethod = self.build.builder.logCompressionMethod 987 self.logs.append(log) 988 for w in self.watchers: 989 receiver = w.logStarted(self.build, self, log) 990 if receiver: 991 log.subscribe(receiver, True) 992 d = log.waitUntilFinished() 993 d.addCallback(lambda log: log.unsubscribe(receiver)) 994 d = log.waitUntilFinished() 995 d.addCallback(self.logFinished) 996 return log

997

998 - def addHTMLLog(self, name, html):

999 assert self.started # addLog before stepStarted won't notify watchers 1000 logfilename = self.build.generateLogfileName(self.name, name) 1001 log = HTMLLogFile(self, name, logfilename, html) 1002 self.logs.append(log) 1003 for w in self.watchers: 1004 receiver = w.logStarted(self.build, self, log) 1005 # TODO: think about this: there isn't much point in letting 1006 # them subscribe 1007 #if receiver: 1008 # log.subscribe(receiver, True) 1009 w.logFinished(self.build, self, log)

1010

1011 - def logFinished(self, log):

1012 for w in self.watchers: 1013 w.logFinished(self.build, self, log)

1014

1015 - def addURL(self, name, url):

1016 self.urls[name] = url

1017

1018 - def setText(self, text):

1019 self.text = text 1020 for w in self.watchers: 1021 w.stepTextChanged(self.build, self, text)

1022 - def setText2(self, text):

1023 self.text2 = text 1024 for w in self.watchers: 1025 w.stepText2Changed(self.build, self, text)

1026

1027 - def setStatistic(self, name, value):

1028 """Set the given statistic. Usually called by subclasses. 1029 """ 1030 self.statistics[name] = value

1031

1032 - def setSkipped(self, skipped):

1033 self.skipped = skipped

1034

1035 - def stepFinished(self, results):

1036 self.finished = util.now() 1037 self.results = results 1038 cld = [] # deferreds for log compression 1039 logCompressionLimit = self.build.builder.logCompressionLimit 1040 for loog in self.logs: 1041 if not loog.isFinished(): 1042 loog.finish() 1043 # if log compression is on, and it's a real LogFile, 1044 # HTMLLogFiles aren't files 1045 if logCompressionLimit is not False and \ 1046 isinstance(loog, LogFile): 1047 if os.path.getsize(loog.getFilename()) > logCompressionLimit: 1048 loog_deferred = loog.compressLog() 1049 if loog_deferred: 1050 cld.append(loog_deferred) 1051 1052 for r in self.updates.keys(): 1053 if self.updates[r] is not None: 1054 self.updates[r].cancel() 1055 del self.updates[r] 1056 1057 watchers = self.finishedWatchers 1058 self.finishedWatchers = [] 1059 for w in watchers: 1060 w.callback(self) 1061 if cld: 1062 return defer.DeferredList(cld)

1063

1064 - def checkLogfiles(self):

1065 # filter out logs that have been deleted 1066 self.logs = [ l for l in self.logs if l.hasContents() ]

1067

1068 - def isWaitingForLocks(self):

1069 return self.waitingForLocks

1070

1071 - def setWaitingForLocks(self, waiting):

1072 self.waitingForLocks = waiting

1073 1074 # persistence 1075

1076 - def __getstate__(self):

1077 d = styles.Versioned.__getstate__(self) 1078 del d['build'] # filled in when loading 1079 if d.has_key('progress'): 1080 del d['progress'] 1081 del d['watchers'] 1082 del d['finishedWatchers'] 1083 del d['updates'] 1084 return d

1085

1086 - def __setstate__(self, d):

1087 styles.Versioned.__setstate__(self, d) 1088 # self.build must be filled in by our parent 1089 1090 # point the logs to this object 1091 for loog in self.logs: 1092 loog.step = self

1093

1094 - def upgradeToVersion1(self):

1095 if not hasattr(self, "urls"): 1096 self.urls = {}

1097

1098 - def upgradeToVersion2(self):

1099 if not hasattr(self, "statistics"): 1100 self.statistics = {}

1101

1102 - def upgradeToVersion3(self):

1103 if not hasattr(self, "step_number"): 1104 self.step_number = 0

1105

1106 - def asDict(self):

1107 result = {} 1108 # Constant 1109 result['name'] = self.getName() 1110 1111 # Transient 1112 result['text'] = self.getText() 1113 result['results'] = self.getResults() 1114 result['isStarted'] = self.isStarted() 1115 result['isFinished'] = self.isFinished() 1116 result['statistics'] = self.statistics 1117 result['times'] = self.getTimes() 1118 result['expectations'] = self.getExpectations() 1119 result['eta'] = self.getETA() 1120 result['urls'] = self.getURLs() 1121 result['step_number'] = self.step_number 1122 # TODO(maruel): Move that to a sub-url or just publish the log_url 1123 # instead. 1124 #result['logs'] = self.getLogs() 1125 return result

1126

1127 1128 -class BuildStatus(styles.Versioned):

1129 implements(interfaces.IBuildStatus, interfaces.IStatusEvent) 1130 persistenceVersion = 3 1131 1132 source = None 1133 reason = None 1134 changes = [] 1135 blamelist = [] 1136 progress = None 1137 started = None 1138 finished = None 1139 currentStep = None 1140 text = [] 1141 results = None 1142 slavename = "???" 1143 1144 # these lists/dicts are defined here so that unserialized instances have 1145 # (empty) values. They are set in __init__ to new objects to make sure 1146 # each instance gets its own copy. 1147 watchers = [] 1148 updates = {} 1149 finishedWatchers = [] 1150 testResults = {} 1151

1152 - def __init__(self, parent, number):

1153 """ 1154 @type parent: L{BuilderStatus} 1155 @type number: int 1156 """ 1157 assert interfaces.IBuilderStatus(parent) 1158 self.builder = parent 1159 self.number = number 1160 self.watchers = [] 1161 self.updates = {} 1162 self.finishedWatchers = [] 1163 self.steps = [] 1164 self.testResults = {} 1165 self.properties = Properties()

1166

1167 - def __repr__(self):

1168 return "<%s #%s>" % (self.__class__.__name__, self.number)

1169 1170 # IBuildStatus 1171

1172 - def getBuilder(self):

1173 """ 1174 @rtype: L{BuilderStatus} 1175 """ 1176 return self.builder

1177

1178 - def getProperty(self, propname):

1179 return self.properties[propname]

1180

1181 - def getProperties(self):

1182 return self.properties

1183

1184 - def getNumber(self):

1185 return self.number

1186

1187 - def getPreviousBuild(self):

1188 if self.number == 0: 1189 return None 1190 return self.builder.getBuild(self.number-1)

1191

1192 - def getSourceStamp(self, absolute=False):

1193 if not absolute or not self.properties.has_key('got_revision'): 1194 return self.source 1195 return self.source.getAbsoluteSourceStamp(self.properties['got_revision'])

1196

1197 - def getReason(self):

1198 return self.reason

1199

1200 - def getChanges(self):

1201 return self.changes

1202

1203 - def getResponsibleUsers(self):

1204 return self.blamelist

1205

1206 - def getInterestedUsers(self):

1207 # TODO: the Builder should add others: sheriffs, domain-owners 1208 return self.blamelist + self.properties.getProperty('owners', [])

1209

1210 - def getSteps(self):

1211 """Return a list of IBuildStepStatus objects. For invariant builds 1212 (those which always use the same set of Steps), this should be the 1213 complete list, however some of the steps may not have started yet 1214 (step.getTimes()[0] will be None). For variant builds, this may not 1215 be complete (asking again later may give you more of them).""" 1216 return self.steps

1217

1218 - def getTimes(self):

1219 return (self.started, self.finished)

1220 1221 _sentinel = [] # used as a sentinel to indicate unspecified initial_value

1222 - def getSummaryStatistic(self, name, summary_fn, initial_value=_sentinel):

1223 """Summarize the named statistic over all steps in which it 1224 exists, using combination_fn and initial_value to combine multiple 1225 results into a single result. This translates to a call to Python's 1226 X{reduce}:: 1227 return reduce(summary_fn, step_stats_list, initial_value) 1228 """ 1229 step_stats_list = [ 1230 st.getStatistic(name) 1231 for st in self.steps 1232 if st.hasStatistic(name) ] 1233 if initial_value is self._sentinel: 1234 return reduce(summary_fn, step_stats_list) 1235 else: 1236 return reduce(summary_fn, step_stats_list, initial_value)

1237

1238 - def isFinished(self):

1239 return (self.finished is not None)

1240

1241 - def waitUntilFinished(self):

1242 if self.finished: 1243 d = defer.succeed(self) 1244 else: 1245 d = defer.Deferred() 1246 self.finishedWatchers.append(d) 1247 return d

1248 1249 # while the build is running, the following methods make sense. 1250 # Afterwards they return None 1251

1252 - def getETA(self):

1253 if self.finished is not None: 1254 return None 1255 if not self.progress: 1256 return None 1257 eta = self.progress.eta() 1258 if eta is None: 1259 return None 1260 return eta - util.now()

1261

1262 - def getCurrentStep(self):

1263 return self.currentStep

1264 1265 # Once you know the build has finished, the following methods are legal. 1266 # Before ths build has finished, they all return None. 1267

1268 - def getText(self):

1269 text = [] 1270 text.extend(self.text) 1271 for s in self.steps: 1272 text.extend(s.text2) 1273 return text

1274

1275 - def getResults(self):

1276 return self.results

1277

1278 - def getSlavename(self):

1279 return self.slavename

1280

1281 - def getTestResults(self):

1282 return self.testResults

1283

1284 - def getTestResultsOrd(self):

1285 trs = self.testResults.keys() 1286 trs.sort() 1287 ret = [ self.testResults[t] for t in trs] 1288 return ret

1289

1290 - def getLogs(self):

1291 # TODO: steps should contribute significant logs instead of this 1292 # hack, which returns every log from every step. The logs should get 1293 # names like "compile" and "test" instead of "compile.output" 1294 logs = [] 1295 for s in self.steps: 1296 for log in s.getLogs(): 1297 logs.append(log) 1298 return logs

1299 1300 # subscription interface 1301 1308

1309 - def sendETAUpdate(self, receiver, updateInterval):

1310 self.updates[receiver] = None 1311 ETA = self.getETA() 1312 if ETA is not None: 1313 receiver.buildETAUpdate(self, self.getETA()) 1314 # they might have unsubscribed during buildETAUpdate 1315 if receiver in self.watchers: 1316 self.updates[receiver] = reactor.callLater(updateInterval, 1317 self.sendETAUpdate, 1318 receiver, 1319 updateInterval)

1320

1321 - def unsubscribe(self, receiver):

1322 if receiver in self.watchers: 1323 self.watchers.remove(receiver) 1324 if receiver in self.updates: 1325 if self.updates[receiver] is not None: 1326 self.updates[receiver].cancel() 1327 del self.updates[receiver]

1328 1329 # methods for the base.Build to invoke 1330

1331 - def addStepWithName(self, name):

1332 """The Build is setting up, and has added a new BuildStep to its 1333 list. Create a BuildStepStatus object to which it can send status 1334 updates.""" 1335 1336 s = BuildStepStatus(self, len(self.steps)) 1337 s.setName(name) 1338 self.steps.append(s) 1339 return s

1340

1341 - def setProperty(self, propname, value, source, runtime=True):

1342 self.properties.setProperty(propname, value, source, runtime)

1343

1344 - def addTestResult(self, result):

1345 self.testResults[result.getName()] = result

1346

1347 - def setSourceStamp(self, sourceStamp):

1348 self.source = sourceStamp 1349 self.changes = self.source.changes

1350

1351 - def setReason(self, reason):

1352 self.reason = reason

1353 - def setBlamelist(self, blamelist):

1354 self.blamelist = blamelist

1355 - def setProgress(self, progress):

1356 self.progress = progress

1357

1358 - def buildStarted(self, build):

1359 """The Build has been set up and is about to be started. It can now 1360 be safely queried, so it is time to announce the new build.""" 1361 1362 self.started = util.now() 1363 # now that we're ready to report status, let the BuilderStatus tell 1364 # the world about us 1365 self.builder.buildStarted(self)

1366

1367 - def setSlavename(self, slavename):

1368 self.slavename = slavename

1369

1370 - def setText(self, text):

1371 assert isinstance(text, (list, tuple)) 1372 self.text = text

1373 - def setResults(self, results):

1374 self.results = results

1375

1376 - def buildFinished(self):

1377 self.currentStep = None 1378 self.finished = util.now() 1379 1380 for r in self.updates.keys(): 1381 if self.updates[r] is not None: 1382 self.updates[r].cancel() 1383 del self.updates[r] 1384 1385 watchers = self.finishedWatchers 1386 self.finishedWatchers = [] 1387 for w in watchers: 1388 w.callback(self)

1389 1390 # methods called by our BuildStepStatus children 1391

1392 - def stepStarted(self, step):

1393 self.currentStep = step 1394 for w in self.watchers: 1395 receiver = w.stepStarted(self, step) 1396 if receiver: 1397 if type(receiver) == type(()): 1398 step.subscribe(receiver[0], receiver[1]) 1399 else: 1400 step.subscribe(receiver) 1401 d = step.waitUntilFinished() 1402 d.addCallback(lambda step: step.unsubscribe(receiver)) 1403 1404 step.waitUntilFinished().addCallback(self._stepFinished)

1405

1406 - def _stepFinished(self, step):

1407 results = step.getResults() 1408 for w in self.watchers: 1409 w.stepFinished(self, step, results)

1410 1411 # methods called by our BuilderStatus parent 1412

1413 - def pruneSteps(self):

1414 # this build is very old: remove the build steps too 1415 self.steps = []

1416 1417 # persistence stuff 1418

1419 - def generateLogfileName(self, stepname, logname):

1420 """Return a filename (relative to the Builder's base directory) where 1421 the logfile's contents can be stored uniquely. 1422 1423 The base filename is made by combining our build number, the Step's 1424 name, and the log's name, then removing unsuitable characters. The 1425 filename is then made unique by appending _0, _1, etc, until it does 1426 not collide with any other logfile. 1427 1428 These files are kept in the Builder's basedir (rather than a 1429 per-Build subdirectory) because that makes cleanup easier: cron and 1430 find will help get rid of the old logs, but the empty directories are 1431 more of a hassle to remove.""" 1432 1433 starting_filename = "%d-log-%s-%s" % (self.number, stepname, logname) 1434 starting_filename = re.sub(r'[^\w\.\-]', '_', starting_filename) 1435 # now make it unique 1436 unique_counter = 0 1437 filename = starting_filename 1438 while filename in [l.filename 1439 for step in self.steps 1440 for l in step.getLogs() 1441 if l.filename]: 1442 filename = "%s_%d" % (starting_filename, unique_counter) 1443 unique_counter += 1 1444 return filename

1445

1446 - def __getstate__(self):

1447 d = styles.Versioned.__getstate__(self) 1448 # for now, a serialized Build is always "finished". We will never 1449 # save unfinished builds. 1450 if not self.finished: 1451 d['finished'] = True 1452 # TODO: push an "interrupted" step so it is clear that the build 1453 # was interrupted. The builder will have a 'shutdown' event, but 1454 # someone looking at just this build will be confused as to why 1455 # the last log is truncated. 1456 for k in 'builder', 'watchers', 'updates', 'finishedWatchers': 1457 if k in d: del d[k] 1458 return d

1459

1460 - def __setstate__(self, d):

1461 styles.Versioned.__setstate__(self, d) 1462 # self.builder must be filled in by our parent when loading 1463 for step in self.steps: 1464 step.build = self 1465 self.watchers = [] 1466 self.updates = {} 1467 self.finishedWatchers = []

1468

1469 - def upgradeToVersion1(self):

1470 if hasattr(self, "sourceStamp"): 1471 # the old .sourceStamp attribute wasn't actually very useful 1472 maxChangeNumber, patch = self.sourceStamp 1473 changes = getattr(self, 'changes', []) 1474 source = sourcestamp.SourceStamp(branch=None, 1475 revision=None, 1476 patch=patch, 1477 changes=changes) 1478 self.source = source 1479 self.changes = source.changes 1480 del self.sourceStamp

1481

1482 - def upgradeToVersion2(self):

1483 self.properties = {}

1484

1485 - def upgradeToVersion3(self):

1486 # in version 3, self.properties became a Properties object 1487 propdict = self.properties 1488 self.properties = Properties() 1489 self.properties.update(propdict, "Upgrade from previous version")

1490

1491 - def upgradeLogfiles(self):

1492 # upgrade any LogFiles that need it. This must occur after we've been 1493 # attached to our Builder, and after we know about all LogFiles of 1494 # all Steps (to get the filenames right). 1495 assert self.builder 1496 for s in self.steps: 1497 for l in s.getLogs(): 1498 if l.filename: 1499 pass # new-style, log contents are on disk 1500 else: 1501 logfilename = self.generateLogfileName(s.name, l.name) 1502 # let the logfile update its .filename pointer, 1503 # transferring its contents onto disk if necessary 1504 l.upgrade(logfilename)

1505

1506 - def checkLogfiles(self):

1507 # check that all logfiles exist, and remove references to any that 1508 # have been deleted (e.g., by purge()) 1509 for s in self.steps: 1510 s.checkLogfiles()

1511

1512 - def saveYourself(self):

1513 filename = os.path.join(self.builder.basedir, "%d" % self.number) 1514 if os.path.isdir(filename): 1515 # leftover from 0.5.0, which stored builds in directories 1516 shutil.rmtree(filename, ignore_errors=True) 1517 tmpfilename = filename + ".tmp" 1518 try: 1519 dump(self, open(tmpfilename, "wb"), -1) 1520 if runtime.platformType == 'win32': 1521 # windows cannot rename a file on top of an existing one, so 1522 # fall back to delete-first. There are ways this can fail and 1523 # lose the builder's history, so we avoid using it in the 1524 # general (non-windows) case 1525 if os.path.exists(filename): 1526 os.unlink(filename) 1527 os.rename(tmpfilename, filename) 1528 except: 1529 log.msg("unable to save build %s-#%d" % (self.builder.name, 1530 self.number)) 1531 log.err()

1532

1533 - def asDict(self):

1534 result = {} 1535 # Constant 1536 result['builderName'] = self.builder.name 1537 result['number'] = self.getNumber() 1538 result['sourceStamp'] = self.getSourceStamp().asDict() 1539 result['reason'] = self.getReason() 1540 result['blame'] = self.getResponsibleUsers() 1541 1542 # Transient 1543 result['properties'] = self.getProperties().asList() 1544 result['times'] = self.getTimes() 1545 result['text'] = self.getText() 1546 result['results'] = self.getResults() 1547 result['slave'] = self.getSlavename() 1548 # TODO(maruel): Add. 1549 #result['test_results'] = self.getTestResults() 1550 # TODO(maruel): Include the url? It's too heavy otherwise. 1551 #result['logs'] = self.getLogs() 1552 result['eta'] = self.getETA() 1553 result['steps'] = [bss.asDict() for bss in self.steps] 1554 if self.getCurrentStep(): 1555 result['currentStep'] = self.getCurrentStep().asDict() 1556 else: 1557 result['currentStep'] = None 1558 return result

1559

1560 1561 1562 -class BuilderStatus(styles.Versioned):

1563 """I handle status information for a single process.base.Builder object. 1564 That object sends status changes to me (frequently as Events), and I 1565 provide them on demand to the various status recipients, like the HTML 1566 waterfall display and the live status clients. It also sends build 1567 summaries to me, which I log and provide to status clients who aren't 1568 interested in seeing details of the individual build steps. 1569 1570 I am responsible for maintaining the list of historic Events and Builds, 1571 pruning old ones, and loading them from / saving them to disk. 1572 1573 I live in the buildbot.process.base.Builder object, in the 1574 .builder_status attribute. 1575 1576 @type category: string 1577 @ivar category: user-defined category this builder belongs to; can be 1578 used to filter on in status clients 1579 """ 1580 1581 implements(interfaces.IBuilderStatus, interfaces.IEventSource) 1582 persistenceVersion = 1 1583 1584 # these limit the amount of memory we consume, as well as the size of the 1585 # main Builder pickle. The Build and LogFile pickles on disk must be 1586 # handled separately. 1587 buildCacheSize = 15 1588 eventHorizon = 50 # forget events beyond this 1589 1590 # these limit on-disk storage 1591 logHorizon = 40 # forget logs in steps in builds beyond this 1592 buildHorizon = 100 # forget builds beyond this 1593 1594 category = None 1595 currentBigState = "offline" # or idle/waiting/interlocked/building 1596 basedir = None # filled in by our parent 1597

1598 - def __init__(self, buildername, category=None):

1599 self.name = buildername 1600 self.category = category 1601 1602 self.slavenames = [] 1603 self.events = [] 1604 # these three hold Events, and are used to retrieve the current 1605 # state of the boxes. 1606 self.lastBuildStatus = None 1607 #self.currentBig = None 1608 #self.currentSmall = None 1609 self.currentBuilds = [] 1610 self.nextBuild = None 1611 self.watchers = [] 1612 self.buildCache = weakref.WeakValueDictionary() 1613 self.buildCache_LRU = [] 1614 self.logCompressionLimit = False # default to no compression for tests 1615 self.logCompressionMethod = "bz2" 1616 self.logMaxSize = None # No default limit 1617 self.logMaxTailSize = None # No tail buffering

1618 1619 # persistence 1620

1621 - def __getstate__(self):

1622 # when saving, don't record transient stuff like what builds are 1623 # currently running, because they won't be there when we start back 1624 # up. Nor do we save self.watchers, nor anything that gets set by our 1625 # parent like .basedir and .status 1626 d = styles.Versioned.__getstate__(self) 1627 d['watchers'] = [] 1628 del d['buildCache'] 1629 del d['buildCache_LRU'] 1630 for b in self.currentBuilds: 1631 b.saveYourself() 1632 # TODO: push a 'hey, build was interrupted' event 1633 del d['currentBuilds'] 1634 d.pop('pendingBuilds', None) 1635 del d['currentBigState'] 1636 del d['basedir'] 1637 del d['status'] 1638 del d['nextBuildNumber'] 1639 return d

1640

1641 - def __setstate__(self, d):

1642 # when loading, re-initialize the transient stuff. Remember that 1643 # upgradeToVersion1 and such will be called after this finishes. 1644 styles.Versioned.__setstate__(self, d) 1645 self.buildCache = weakref.WeakValueDictionary() 1646 self.buildCache_LRU = [] 1647 self.currentBuilds = [] 1648 self.watchers = [] 1649 self.slavenames = []

1650 # self.basedir must be filled in by our parent 1651 # self.status must be filled in by our parent 1652

1653 - def reconfigFromBuildmaster(self, buildmaster):

1654 # Note that we do not hang onto the buildmaster, since this object 1655 # gets pickled and unpickled. 1656 if buildmaster.buildCacheSize: 1657 self.buildCacheSize = buildmaster.buildCacheSize

1658

1659 - def upgradeToVersion1(self):

1660 if hasattr(self, 'slavename'): 1661 self.slavenames = [self.slavename] 1662 del self.slavename 1663 if hasattr(self, 'nextBuildNumber'): 1664 del self.nextBuildNumber # determineNextBuildNumber chooses this

1665

1666 - def determineNextBuildNumber(self):

1667 """Scan our directory of saved BuildStatus instances to determine 1668 what our self.nextBuildNumber should be. Set it one larger than the 1669 highest-numbered build we discover. This is called by the top-level 1670 Status object shortly after we are created or loaded from disk. 1671 """ 1672 existing_builds = [int(f) 1673 for f in os.listdir(self.basedir) 1674 if re.match("^\d+$", f)] 1675 if existing_builds: 1676 self.nextBuildNumber = max(existing_builds) + 1 1677 else: 1678 self.nextBuildNumber = 0

1679

1680 - def setLogCompressionLimit(self, lowerLimit):

1681 self.logCompressionLimit = lowerLimit

1682

1683 - def setLogCompressionMethod(self, method):

1684 assert method in ("bz2", "gz") 1685 self.logCompressionMethod = method

1686

1687 - def setLogMaxSize(self, upperLimit):

1688 self.logMaxSize = upperLimit

1689

1690 - def setLogMaxTailSize(self, tailSize):

1691 self.logMaxTailSize = tailSize

1692

1693 - def saveYourself(self):

1694 for b in self.currentBuilds: 1695 if not b.isFinished: 1696 # interrupted build, need to save it anyway. 1697 # BuildStatus.saveYourself will mark it as interrupted. 1698 b.saveYourself() 1699 filename = os.path.join(self.basedir, "builder") 1700 tmpfilename = filename + ".tmp" 1701 try: 1702 dump(self, open(tmpfilename, "wb"), -1) 1703 if runtime.platformType == 'win32': 1704 # windows cannot rename a file on top of an existing one 1705 if os.path.exists(filename): 1706 os.unlink(filename) 1707 os.rename(tmpfilename, filename) 1708 except: 1709 log.msg("unable to save builder %s" % self.name) 1710 log.err()

1711 1712 1713 # build cache management 1714

1715 - def makeBuildFilename(self, number):

1716 return os.path.join(self.basedir, "%d" % number)

1717

1718 - def touchBuildCache(self, build):

1719 self.buildCache[build.number] = build 1720 if build in self.buildCache_LRU: 1721 self.buildCache_LRU.remove(build) 1722 self.buildCache_LRU = self.buildCache_LRU[-(self.buildCacheSize-1):] + [ build ] 1723 return build

1724

1725 - def getBuildByNumber(self, number):

1726 # first look in currentBuilds 1727 for b in self.currentBuilds: 1728 if b.number == number: 1729 return self.touchBuildCache(b) 1730 1731 # then in the buildCache 1732 if number in self.buildCache: 1733 return self.touchBuildCache(self.buildCache[number]) 1734 1735 # then fall back to loading it from disk 1736 filename = self.makeBuildFilename(number) 1737 try: 1738 log.msg("Loading builder %s's build %d from on-disk pickle" 1739 % (self.name, number)) 1740 build = load(open(filename, "rb")) 1741 styles.doUpgrade() 1742 build.builder = self 1743 # handle LogFiles from after 0.5.0 and before 0.6.5 1744 build.upgradeLogfiles() 1745 # check that logfiles exist 1746 build.checkLogfiles() 1747 return self.touchBuildCache(build) 1748 except IOError: 1749 raise IndexError("no such build %d" % number) 1750 except EOFError: 1751 raise IndexError("corrupted build pickle %d" % number)

1752

1753 - def prune(self, events_only=False):

1754 # begin by pruning our own events 1755 self.events = self.events[-self.eventHorizon:] 1756 1757 if events_only: 1758 return 1759 1760 gc.collect() 1761 1762 # get the horizons straight 1763 if self.buildHorizon: 1764 earliest_build = self.nextBuildNumber - self.buildHorizon 1765 else: 1766 earliest_build = 0 1767 1768 if self.logHorizon: 1769 earliest_log = self.nextBuildNumber - self.logHorizon 1770 else: 1771 earliest_log = 0 1772 1773 if earliest_log < earliest_build: 1774 earliest_log = earliest_build 1775 1776 if earliest_build == 0: 1777 return 1778 1779 # skim the directory and delete anything that shouldn't be there anymore 1780 build_re = re.compile(r"^([0-9]+)$") 1781 build_log_re = re.compile(r"^([0-9]+)-.*$") 1782 # if the directory doesn't exist, bail out here 1783 if not os.path.exists(self.basedir): 1784 return 1785 1786 for filename in os.listdir(self.basedir): 1787 num = None 1788 mo = build_re.match(filename) 1789 is_logfile = False 1790 if mo: 1791 num = int(mo.group(1)) 1792 else: 1793 mo = build_log_re.match(filename) 1794 if mo: 1795 num = int(mo.group(1)) 1796 is_logfile = True 1797 1798 if num is None: continue 1799 if num in self.buildCache: continue 1800 1801 if (is_logfile and num < earliest_log) or num < earliest_build: 1802 pathname = os.path.join(self.basedir, filename) 1803 log.msg("pruning '%s'" % pathname) 1804 try: os.unlink(pathname) 1805 except OSError: pass

1806 1807 # IBuilderStatus methods

1808 - def getName(self):

1809 return self.name

1810

1811 - def getState(self):

1812 return (self.currentBigState, self.currentBuilds)

1813

1814 - def getSlaves(self):

1815 return [self.status.getSlave(name) for name in self.slavenames]

1816

1817 - def getPendingBuilds(self):

1818 db = self.status.db 1819 return [BuildRequestStatus(brid, self.status, db) 1820 for brid in db.get_pending_brids_for_builder(self.name)]

1821

1822 - def getCurrentBuilds(self):

1823 return self.currentBuilds

1824

1825 - def getLastFinishedBuild(self):

1826 b = self.getBuild(-1) 1827 if not (b and b.isFinished()): 1828 b = self.getBuild(-2) 1829 return b

1830

1831 - def getCategory(self):

1832 return self.category

1833

1834 - def getBuild(self, number):

1835 if number < 0: 1836 number = self.nextBuildNumber + number 1837 if number < 0 or number >= self.nextBuildNumber: 1838 return None 1839 1840 try: 1841 return self.getBuildByNumber(number) 1842 except IndexError: 1843 return None

1844

1845 - def getEvent(self, number):

1846 try: 1847 return self.events[number] 1848 except IndexError: 1849 return None

1850

1851 - def generateFinishedBuilds(self, branches=[], 1852 num_builds=None, 1853 max_buildnum=None, 1854 finished_before=None, 1855 max_search=200):

1856 got = 0 1857 for Nb in itertools.count(1): 1858 if Nb > self.nextBuildNumber: 1859 break 1860 if Nb > max_search: 1861 break 1862 build = self.getBuild(-Nb) 1863 if build is None: 1864 continue 1865 if max_buildnum is not None: 1866 if build.getNumber() > max_buildnum: 1867 continue 1868 if not build.isFinished(): 1869 continue 1870 if finished_before is not None: 1871 start, end = build.getTimes() 1872 if end >= finished_before: 1873 continue 1874 if branches: 1875 if build.getSourceStamp().branch not in branches: 1876 continue 1877 got += 1 1878 yield build 1879 if num_builds is not None: 1880 if got >= num_builds: 1881 return

1882

1883 - def eventGenerator(self, branches=[], categories=[], committers=[], minTime=0):

1884 """This function creates a generator which will provide all of this 1885 Builder's status events, starting with the most recent and 1886 progressing backwards in time. """ 1887 1888 # remember the oldest-to-earliest flow here. "next" means earlier. 1889 1890 # TODO: interleave build steps and self.events by timestamp. 1891 # TODO: um, I think we're already doing that. 1892 1893 # TODO: there's probably something clever we could do here to 1894 # interleave two event streams (one from self.getBuild and the other 1895 # from self.getEvent), which would be simpler than this control flow 1896 1897 eventIndex = -1 1898 e = self.getEvent(eventIndex) 1899 for Nb in range(1, self.nextBuildNumber+1): 1900 b = self.getBuild(-Nb) 1901 if not b: 1902 # HACK: If this is the first build we are looking at, it is 1903 # possible it's in progress but locked before it has written a 1904 # pickle; in this case keep looking. 1905 if Nb == 1: 1906 continue 1907 break 1908 if b.getTimes()[0] < minTime: 1909 break 1910 if branches and not b.getSourceStamp().branch in branches: 1911 continue 1912 if categories and not b.getBuilder().getCategory() in categories: 1913 continue 1914 if committers and not [True for c in b.getChanges() if c.who in committers]: 1915 continue 1916 steps = b.getSteps() 1917 for Ns in range(1, len(steps)+1): 1918 if steps[-Ns].started: 1919 step_start = steps[-Ns].getTimes()[0] 1920 while e is not None and e.getTimes()[0] > step_start: 1921 yield e 1922 eventIndex -= 1 1923 e = self.getEvent(eventIndex) 1924 yield steps[-Ns] 1925 yield b 1926 while e is not None: 1927 yield e 1928 eventIndex -= 1 1929 e = self.getEvent(eventIndex) 1930 if e and e.getTimes()[0] < minTime: 1931 break

1932 1942

1943 - def unsubscribe(self, receiver):

1944 self.watchers.remove(receiver) 1945 self.status._builder_unsubscribe(self.name, receiver)

1946 1947 ## Builder interface (methods called by the Builder which feeds us) 1948

1949 - def setSlavenames(self, names):

1950 self.slavenames = names

1951

1952 - def addEvent(self, text=[]):

1953 # this adds a duration event. When it is done, the user should call 1954 # e.finish(). They can also mangle it by modifying .text 1955 e = Event() 1956 e.started = util.now() 1957 e.text = text 1958 self.events.append(e) 1959 self.prune(events_only=True) 1960 return e # they are free to mangle it further

1961

1962 - def addPointEvent(self, text=[]):

1963 # this adds a point event, one which occurs as a single atomic 1964 # instant of time. 1965 e = Event() 1966 e.started = util.now() 1967 e.finished = 0 1968 e.text = text 1969 self.events.append(e) 1970 self.prune(events_only=True) 1971 return e # for consistency, but they really shouldn't touch it

1972

1973 - def setBigState(self, state):

1974 needToUpdate = state != self.currentBigState 1975 self.currentBigState = state 1976 if needToUpdate: 1977 self.publishState()

1978

1979 - def publishState(self, target=None):

1980 state = self.currentBigState 1981 1982 if target is not None: 1983 # unicast 1984 target.builderChangedState(self.name, state) 1985 return 1986 for w in self.watchers: 1987 try: 1988 w.builderChangedState(self.name, state) 1989 except: 1990 log.msg("Exception caught publishing state to %r" % w) 1991 log.err()

1992

1993 - def newBuild(self):

1994 """The Builder has decided to start a build, but the Build object is 1995 not yet ready to report status (it has not finished creating the 1996 Steps). Create a BuildStatus object that it can use.""" 1997 number = self.nextBuildNumber 1998 self.nextBuildNumber += 1 1999 # TODO: self.saveYourself(), to make sure we don't forget about the 2000 # build number we've just allocated. This is not quite as important 2001 # as it was before we switch to determineNextBuildNumber, but I think 2002 # it may still be useful to have the new build save itself. 2003 s = BuildStatus(self, number) 2004 s.waitUntilFinished().addCallback(self._buildFinished) 2005 return s

2006 2007 # buildStarted is called by our child BuildStatus instances

2008 - def buildStarted(self, s):

2009 """Now the BuildStatus object is ready to go (it knows all of its 2010 Steps, its ETA, etc), so it is safe to notify our watchers.""" 2011 2012 assert s.builder is self # paranoia 2013 assert s.number == self.nextBuildNumber - 1 2014 assert s not in self.currentBuilds 2015 self.currentBuilds.append(s) 2016 self.touchBuildCache(s) 2017 2018 # now that the BuildStatus is prepared to answer queries, we can 2019 # announce the new build to all our watchers 2020 2021 for w in self.watchers: # TODO: maybe do this later? callLater(0)? 2022 try: 2023 receiver = w.buildStarted(self.getName(), s) 2024 if receiver: 2025 if type(receiver) == type(()): 2026 s.subscribe(receiver[0], receiver[1]) 2027 else: 2028 s.subscribe(receiver) 2029 d = s.waitUntilFinished() 2030 d.addCallback(lambda s: s.unsubscribe(receiver)) 2031 except: 2032 log.msg("Exception caught notifying %r of buildStarted event" % w) 2033 log.err()

2034

2035 - def _buildFinished(self, s):

2036 assert s in self.currentBuilds 2037 s.saveYourself() 2038 self.currentBuilds.remove(s) 2039 2040 name = self.getName() 2041 results = s.getResults() 2042 for w in self.watchers: 2043 try: 2044 w.buildFinished(name, s, results) 2045 except: 2046 log.msg("Exception caught notifying %r of buildFinished event" % w) 2047 log.err() 2048 2049 self.prune() # conserve disk

2050 2051 2052 # waterfall display (history) 2053 2054 # I want some kind of build event that holds everything about the build: 2055 # why, what changes went into it, the results of the build, itemized 2056 # test results, etc. But, I do kind of need something to be inserted in 2057 # the event log first, because intermixing step events and the larger 2058 # build event is fraught with peril. Maybe an Event-like-thing that 2059 # doesn't have a file in it but does have links. Hmm, that's exactly 2060 # what it does now. The only difference would be that this event isn't 2061 # pushed to the clients. 2062 2063 # publish to clients

2064 - def sendLastBuildStatus(self, client):

2065 #client.newLastBuildStatus(self.lastBuildStatus) 2066 pass

2067 - def sendCurrentActivityBigToEveryone(self):

2068 for s in self.subscribers: 2069 self.sendCurrentActivityBig(s)

2070 - def sendCurrentActivityBig(self, client):

2071 state = self.currentBigState 2072 if state == "offline": 2073 client.currentlyOffline() 2074 elif state == "idle": 2075 client.currentlyIdle() 2076 elif state == "building": 2077 client.currentlyBuilding() 2078 else: 2079 log.msg("Hey, self.currentBigState is weird:", state)

2080 2081 2082 ## HTML display interface 2083

2084 - def getEventNumbered(self, num):

2085 # deal with dropped events, pruned events 2086 first = self.events[0].number 2087 if first + len(self.events)-1 != self.events[-1].number: 2088 log.msg(self, 2089 "lost an event somewhere: [0] is %d, [%d] is %d" % \ 2090 (self.events[0].number, 2091 len(self.events) - 1, 2092 self.events[-1].number)) 2093 for e in self.events: 2094 log.msg("e[%d]: " % e.number, e) 2095 return None 2096 offset = num - first 2097 log.msg(self, "offset", offset) 2098 try: 2099 return self.events[offset] 2100 except IndexError: 2101 return None

2102 2103 ## Persistence of Status

2104 - def loadYourOldEvents(self):

2105 if hasattr(self, "allEvents"): 2106 # first time, nothing to get from file. Note that this is only if 2107 # the Application gets .run() . If it gets .save()'ed, then the 2108 # .allEvents attribute goes away in the initial __getstate__ and 2109 # we try to load a non-existent file. 2110 return 2111 self.allEvents = self.loadFile("events", []) 2112 if self.allEvents: 2113 self.nextEventNumber = self.allEvents[-1].number + 1 2114 else: 2115 self.nextEventNumber = 0

2116 - def saveYourOldEvents(self):

2117 self.saveFile("events", self.allEvents)

2118 2119 ## clients 2120

2121 - def addClient(self, client):

2122 if client not in self.subscribers: 2123 self.subscribers.append(client) 2124 self.sendLastBuildStatus(client) 2125 self.sendCurrentActivityBig(client) 2126 client.newEvent(self.currentSmall)

2127 - def removeClient(self, client):

2128 if client in self.subscribers: 2129 self.subscribers.remove(client)

2130

2131 - def asDict(self):

2132 result = {} 2133 # Constant 2134 # TODO(maruel): Fix me. We don't want to leak the full path. 2135 result['basedir'] = os.path.basename(self.basedir) 2136 result['category'] = self.category 2137 result['slaves'] = self.slavenames 2138 #result['url'] = self.parent.getURLForThing(self) 2139 # TODO(maruel): Add cache settings? Do we care? 2140 2141 # Transient 2142 # Collect build numbers. 2143 # Important: Only grab the *cached* builds numbers to reduce I/O. 2144 current_builds = [b.getNumber() for b in self.currentBuilds] 2145 cached_builds = list(set(self.buildCache.keys() + current_builds)) 2146 cached_builds.sort() 2147 result['cachedBuilds'] = cached_builds 2148 result['currentBuilds'] = current_builds 2149 result['state'] = self.getState()[0] 2150 # BuildRequestStatus doesn't have a number so display the SourceStamp. 2151 result['pendingBuilds'] = [ 2152 b.getSourceStamp().asDict() for b in self.getPendingBuilds() 2153 ] 2154 return result

2155

2156 2157 -class SlaveStatus:

2158 implements(interfaces.ISlaveStatus) 2159 2160 admin = None 2161 host = None 2162 access_uri = None 2163 version = None 2164 connected = False 2165 graceful_shutdown = False 2166

2167 - def __init__(self, name):

2168 self.name = name 2169 self._lastMessageReceived = 0 2170 self.runningBuilds = [] 2171 self.graceful_callbacks = [] 2172 self.connect_times = []

2173

2174 - def getName(self):

2175 return self.name

2176 - def getAdmin(self):

2177 return self.admin

2178 - def getHost(self):

2179 return self.host

2180 - def getAccessURI(self):

2181 return self.access_uri

2182 - def getVersion(self):

2183 return self.version

2184 - def isConnected(self):

2185 return self.connected

2186 - def lastMessageReceived(self):

2187 return self._lastMessageReceived

2188 - def getRunningBuilds(self):

2189 return self.runningBuilds

2190 - def getConnectCount(self):

2191 then = time.time() - 3600 2192 return len([ t for t in self.connect_times if t > then ])

2193

2194 - def setAdmin(self, admin):

2195 self.admin = admin

2196 - def setHost(self, host):

2197 self.host = host

2198 - def setAccessURI(self, access_uri):

2199 self.access_uri = access_uri

2200 - def setVersion(self, version):

2201 self.version = version

2202 - def setConnected(self, isConnected):

2203 self.connected = isConnected

2204 - def setLastMessageReceived(self, when):

2205 self._lastMessageReceived = when

2206

2207 - def recordConnectTime(self):

2208 # record this connnect, and keep data for the last hour 2209 now = time.time() 2210 self.connect_times = [ t for t in self.connect_times if t > now - 3600 ] + [ now ]

2211

2212 - def buildStarted(self, build):

2213 self.runningBuilds.append(build)

2214 - def buildFinished(self, build):

2215 self.runningBuilds.remove(build)

2216

2217 - def getGraceful(self):

2218 """Return the graceful shutdown flag""" 2219 return self.graceful_shutdown

2220 - def setGraceful(self, graceful):

2221 """Set the graceful shutdown flag, and notify all the watchers""" 2222 self.graceful_shutdown = graceful 2223 for cb in self.graceful_callbacks: 2224 eventually(cb, graceful)

2225 - def addGracefulWatcher(self, watcher):

2226 """Add watcher to the list of watchers to be notified when the 2227 graceful shutdown flag is changed.""" 2228 if not watcher in self.graceful_callbacks: 2229 self.graceful_callbacks.append(watcher)

2230 - def removeGracefulWatcher(self, watcher):

2231 """Remove watcher from the list of watchers to be notified when the 2232 graceful shutdown flag is changed.""" 2233 if watcher in self.graceful_callbacks: 2234 self.graceful_callbacks.remove(watcher)

2235

2236 - def asDict(self):

2237 result = {} 2238 # Constant 2239 result['name'] = self.getName() 2240 result['access_uri'] = self.getAccessURI() 2241 2242 # Transient (since it changes when the slave reconnects) 2243 result['host'] = self.getHost() 2244 result['admin'] = self.getAdmin() 2245 result['version'] = self.getVersion() 2246 result['connected'] = self.isConnected() 2247 result['runningBuilds'] = [b.asDict() for b in self.getRunningBuilds()] 2248 return result

2249

2250 -class Status:

2251 """ 2252 I represent the status of the buildmaster. 2253 """ 2254 implements(interfaces.IStatus) 2255

2256 - def __init__(self, botmaster, basedir):

2257 """ 2258 @type botmaster: L{buildbot.master.BotMaster} 2259 @param botmaster: the Status object uses C{.botmaster} to get at 2260 both the L{buildbot.master.BuildMaster} (for 2261 various buildbot-wide parameters) and the 2262 actual Builders (to get at their L{BuilderStatus} 2263 objects). It is not allowed to change or influence 2264 anything through this reference. 2265 @type basedir: string 2266 @param basedir: this provides a base directory in which saved status 2267 information (changes.pck, saved Build status 2268 pickles) can be stored 2269 """ 2270 self.botmaster = botmaster 2271 self.db = None 2272 self.basedir = basedir 2273 self.watchers = [] 2274 assert os.path.isdir(basedir) 2275 # compress logs bigger than 4k, a good default on linux 2276 self.logCompressionLimit = 4*1024 2277 self.logCompressionMethod = "bz2" 2278 # No default limit to the log size 2279 self.logMaxSize = None 2280 self.logMaxTailSize = None 2281 2282 self._builder_observers = collections.KeyedSets() 2283 self._buildreq_observers = collections.KeyedSets() 2284 self._buildset_success_waiters = collections.KeyedSets() 2285 self._buildset_finished_waiters = collections.KeyedSets()

2286 2287 @property

2288 - def shuttingDown(self):

2289 return self.botmaster.shuttingDown

2290

2291 - def cleanShutdown(self):

2292 return self.botmaster.cleanShutdown()

2293

2294 - def cancelCleanShutdown(self):

2295 return self.botmaster.cancelCleanShutdown()

2296

2297 - def setDB(self, db):

2298 self.db = db 2299 self.db.subscribe_to("add-build", self._db_builds_changed) 2300 self.db.subscribe_to("add-buildset", self._db_buildset_added) 2301 self.db.subscribe_to("modify-buildset", self._db_buildsets_changed) 2302 self.db.subscribe_to("add-buildrequest", self._db_buildrequest_added) 2303 self.db.subscribe_to("cancel-buildrequest", self._db_buildrequest_cancelled)

2304 2305 # methods called by our clients 2306

2307 - def getProjectName(self):

2308 return self.botmaster.parent.projectName

2309 - def getProjectURL(self):

2310 return self.botmaster.parent.projectURL

2311 - def getBuildbotURL(self):

2312 return self.botmaster.parent.buildbotURL

2313

2314 - def getURLForThing(self, thing):

2315 prefix = self.getBuildbotURL() 2316 if not prefix: 2317 return None 2318 if interfaces.IStatus.providedBy(thing): 2319 return prefix 2320 if interfaces.ISchedulerStatus.providedBy(thing): 2321 pass 2322 if interfaces.IBuilderStatus.providedBy(thing): 2323 builder = thing 2324 return prefix + "builders/%s" % ( 2325 urllib.quote(builder.getName(), safe=''), 2326 ) 2327 if interfaces.IBuildStatus.providedBy(thing): 2328 build = thing 2329 builder = build.getBuilder() 2330 return prefix + "builders/%s/builds/%d" % ( 2331 urllib.quote(builder.getName(), safe=''), 2332 build.getNumber()) 2333 if interfaces.IBuildStepStatus.providedBy(thing): 2334 step = thing 2335 build = step.getBuild() 2336 builder = build.getBuilder() 2337 return prefix + "builders/%s/builds/%d/steps/%s" % ( 2338 urllib.quote(builder.getName(), safe=''), 2339 build.getNumber(), 2340 urllib.quote(step.getName(), safe='')) 2341 # IBuildSetStatus 2342 # IBuildRequestStatus 2343 # ISlaveStatus 2344 2345 # IStatusEvent 2346 if interfaces.IStatusEvent.providedBy(thing): 2347 from buildbot.changes import changes 2348 # TODO: this is goofy, create IChange or something 2349 if isinstance(thing, changes.Change): 2350 change = thing 2351 return "%schanges/%d" % (prefix, change.number) 2352 2353 if interfaces.IStatusLog.providedBy(thing): 2354 log = thing 2355 step = log.getStep() 2356 build = step.getBuild() 2357 builder = build.getBuilder() 2358 2359 logs = step.getLogs() 2360 for i in range(len(logs)): 2361 if log is logs[i]: 2362 break 2363 else: 2364 return None 2365 return prefix + "builders/%s/builds/%d/steps/%s/logs/%s" % ( 2366 urllib.quote(builder.getName(), safe=''), 2367 build.getNumber(), 2368 urllib.quote(step.getName(), safe=''), 2369 urllib.quote(log.getName()))

2370

2371 - def getChangeSources(self):

2372 return list(self.botmaster.parent.change_svc)

2373

2374 - def getChange(self, number):

2375 return self.botmaster.parent.change_svc.getChangeNumberedNow(number)

2376

2377 - def getSchedulers(self):

2378 return self.botmaster.parent.allSchedulers()

2379

2380 - def getBuilderNames(self, categories=None):

2381 if categories == None: 2382 return self.botmaster.builderNames[:] # don't let them break it 2383 2384 l = [] 2385 # respect addition order 2386 for name in self.botmaster.builderNames: 2387 builder = self.botmaster.builders[name] 2388 if builder.builder_status.category in categories: 2389 l.append(name) 2390 return l

2391

2392 - def getBuilder(self, name):

2393 """ 2394 @rtype: L{BuilderStatus} 2395 """ 2396 return self.botmaster.builders[name].builder_status

2397

2398 - def getSlaveNames(self):

2399 return self.botmaster.slaves.keys()

2400

2401 - def getSlave(self, slavename):

2402 return self.botmaster.slaves[slavename].slave_status

2403

2404 - def getBuildSets(self):

2405 return [BuildSetStatus(bsid, self, self.db) 2406 for bsid in self.db.get_active_buildset_ids()]

2407

2408 - def generateFinishedBuilds(self, builders=[], branches=[], 2409 num_builds=None, finished_before=None, 2410 max_search=200):

2411 2412 def want_builder(bn): 2413 if builders: 2414 return bn in builders 2415 return True

2416 builder_names = [bn 2417 for bn in self.getBuilderNames() 2418 if want_builder(bn)] 2419 2420 # 'sources' is a list of generators, one for each Builder we're 2421 # using. When the generator is exhausted, it is replaced in this list 2422 # with None. 2423 sources = [] 2424 for bn in builder_names: 2425 b = self.getBuilder(bn) 2426 g = b.generateFinishedBuilds(branches, 2427 finished_before=finished_before, 2428 max_search=max_search) 2429 sources.append(g) 2430 2431 # next_build the next build from each source 2432 next_build = [None] * len(sources) 2433 2434 def refill(): 2435 for i,g in enumerate(sources): 2436 if next_build[i]: 2437 # already filled 2438 continue 2439 if not g: 2440 # already exhausted 2441 continue 2442 try: 2443 next_build[i] = g.next() 2444 except StopIteration: 2445 next_build[i] = None 2446 sources[i] = None

2447 2448 got = 0 2449 while True: 2450 refill() 2451 # find the latest build among all the candidates 2452 candidates = [(i, b, b.getTimes()[1]) 2453 for i,b in enumerate(next_build) 2454 if b is not None] 2455 candidates.sort(lambda x,y: cmp(x[2], y[2])) 2456 if not candidates: 2457 return 2458 2459 # and remove it from the list 2460 i, build, finshed_time = candidates[-1] 2461 next_build[i] = None 2462 got += 1 2463 yield build 2464 if num_builds is not None: 2465 if got >= num_builds: 2466 return 2467

2472 - def unsubscribe(self, target):

2473 self.watchers.remove(target)

2474 2475 2476 # methods called by upstream objects 2477

2478 - def announceNewBuilder(self, target, name, builder_status):

2479 t = target.builderAdded(name, builder_status) 2480 if t: 2481 builder_status.subscribe(t)

2482

2483 - def builderAdded(self, name, basedir, category=None):

2484 """ 2485 @rtype: L{BuilderStatus} 2486 """ 2487 filename = os.path.join(self.basedir, basedir, "builder") 2488 log.msg("trying to load status pickle from %s" % filename) 2489 builder_status = None 2490 try: 2491 builder_status = load(open(filename, "rb")) 2492 styles.doUpgrade() 2493 except IOError: 2494 log.msg("no saved status pickle, creating a new one") 2495 except: 2496 log.msg("error while loading status pickle, creating a new one") 2497 log.msg("error follows:") 2498 log.err() 2499 if not builder_status: 2500 builder_status = BuilderStatus(name, category) 2501 builder_status.addPointEvent(["builder", "created"]) 2502 log.msg("added builder %s in category %s" % (name, category)) 2503 # an unpickled object might not have category set from before, 2504 # so set it here to make sure 2505 builder_status.category = category 2506 builder_status.basedir = os.path.join(self.basedir, basedir) 2507 builder_status.name = name # it might have been updated 2508 builder_status.status = self 2509 2510 if not os.path.isdir(builder_status.basedir): 2511 os.makedirs(builder_status.basedir) 2512 builder_status.determineNextBuildNumber() 2513 2514 builder_status.setBigState("offline") 2515 builder_status.setLogCompressionLimit(self.logCompressionLimit) 2516 builder_status.setLogCompressionMethod(self.logCompressionMethod) 2517 builder_status.setLogMaxSize(self.logMaxSize) 2518 builder_status.setLogMaxTailSize(self.logMaxTailSize) 2519 2520 for t in self.watchers: 2521 self.announceNewBuilder(t, name, builder_status) 2522 2523 return builder_status

2524

2525 - def builderRemoved(self, name):

2526 for t in self.watchers: 2527 if hasattr(t, 'builderRemoved'): 2528 t.builderRemoved(name)

2529

2530 - def slaveConnected(self, name):

2531 for t in self.watchers: 2532 if hasattr(t, 'slaveConnected'): 2533 t.slaveConnected(name)

2534

2535 - def slaveDisconnected(self, name):

2536 for t in self.watchers: 2537 if hasattr(t, 'slaveDisconnected'): 2538 t.slaveDisconnected(name)

2539

2540 - def changeAdded(self, change):

2541 for t in self.watchers: 2542 if hasattr(t, 'changeAdded'): 2543 t.changeAdded(change)

2544

2545 - def asDict(self):

2546 result = {} 2547 # Constant 2548 result['projectName'] = self.getProjectName() 2549 result['projectURL'] = self.getProjectURL() 2550 result['buildbotURL'] = self.getBuildbotURL() 2551 # TODO: self.getSchedulers() 2552 # self.getChangeSources() 2553 return result

2554

2555 - def buildreqs_retired(self, requests):

2556 for r in requests: 2557 #r.id: notify subscribers (none right now) 2558 # r.bsid: check for completion, notify subscribers, unsubscribe 2559 pass

2560

2561 - def get_buildreq_for_id(self, brid):

2562 return BuildRequestStatus(brid, self, self.db)

2563

2564 - def _db_builds_changed(self, category, bid):

2565 brid,buildername,buildnum = self.db.get_build_info(bid) 2566 if brid in self._buildreq_observers: 2567 bs = self.getBuilder(buildername).getBuild(buildnum) 2568 if bs: 2569 for o in self._buildreq_observers[brid]: 2570 eventually(o, bs)

2571

2572 - def _buildrequest_subscribe(self, brid, observer):

2573 self._buildreq_observers.add(brid, observer)

2574

2575 - def _buildrequest_unsubscribe(self, brid, observer):

2576 self._buildreq_observers.discard(brid, observer)

2577

2578 - def _db_buildset_added(self, category, bsid):

2579 bss = BuildSetStatus(bsid, self, self.db) 2580 for t in self.watchers: 2581 if hasattr(t, 'buildsetSubmitted'): 2582 t.buildsetSubmitted(bss)

2583

2584 - def _buildset_waitUntilSuccess(self, bsid):

2585 d = defer.Deferred() 2586 self._buildset_success_waiters.add(bsid, d) 2587 # now check for a buildset which was already successful 2588 self._db_buildsets_changed("modify-buildset", bsid) 2589 return d

2590 - def _buildset_waitUntilFinished(self, bsid):

2591 d = defer.Deferred() 2592 self._buildset_finished_waiters.add(bsid, d) 2593 self._db_buildsets_changed("modify-buildset", bsid) 2594 return d

2595

2596 - def _db_buildsets_changed(self, category, *bsids):

2597 for bsid in bsids: 2598 self._db_buildset_changed(bsid)

2599

2600 - def _db_buildset_changed(self, bsid):

2601 # check bsid to see if it's successful or finished, and notify anyone 2602 # who cares 2603 if (bsid not in self._buildset_success_waiters 2604 and bsid not in self._buildset_finished_waiters): 2605 return 2606 successful,finished = self.db.examine_buildset(bsid) 2607 bss = BuildSetStatus(bsid, self, self.db) 2608 if successful is not None: 2609 for d in self._buildset_success_waiters.pop(bsid): 2610 eventually(d.callback, bss) 2611 if finished: 2612 for d in self._buildset_finished_waiters.pop(bsid): 2613 eventually(d.callback, bss)

2614

2615 - def _builder_subscribe(self, buildername, watcher):

2616 # should get requestSubmitted and requestCancelled 2617 self._builder_observers.add(buildername, watcher)

2618

2619 - def _builder_unsubscribe(self, buildername, watcher):

2620 self._builder_observers.discard(buildername, watcher)

2621

2622 - def _db_buildrequest_added(self, category, *brids):

2623 self._handle_buildrequest_event("added", brids)

2624 - def _db_buildrequest_cancelled(self, category, *brids):

2625 self._handle_buildrequest_event("cancelled", brids)

2626 - def _handle_buildrequest_event(self, mode, brids):

2627 for brid in brids: 2628 buildername = self.db.get_buildername_for_brid(brid) 2629 if buildername in self._builder_observers: 2630 brs = BuildRequestStatus(brid, self, self.db) 2631 for observer in self._builder_observers[buildername]: 2632 if mode == "added": 2633 if hasattr(observer, 'requestSubmitted'): 2634 eventually(observer.requestSubmitted, brs) 2635 else: 2636 if hasattr(observer, 'requestCancelled'): 2637 builder = self.getBuilder(buildername) 2638 eventually(observer.requestCancelled, builder, brs)

2639 2640 # vim: set ts=4 sts=4 sw=4 et: 2641

Source Code for Module buildbot.status.builder