buildbot.status.builder

1 # -*- test-case-name: buildbot.test.test_status -*- 2 3 from zope.interface import implements 4 from twisted.python import log 5 from twisted.persisted import styles 6 from twisted.internet import reactor, defer, threads 7 from twisted.protocols import basic 8 from buildbot.process.properties import Properties 9 from buildbot.util import collections 10 from buildbot.util.eventual import eventually 11 12 import weakref 13 import os, shutil, sys, re, urllib, itertools 14 import gc 15 from cPickle import load, dump 16 from cStringIO import StringIO 17 from bz2 import BZ2File 18 from gzip import GzipFile 19 20 # sibling imports 21 from buildbot import interfaces, util, sourcestamp 22 23 SUCCESS, WARNINGS, FAILURE, SKIPPED, EXCEPTION, RETRY = range(6) 24 Results = ["success", "warnings", "failure", "skipped", "exception", "retry"] 25 26 27 # build processes call the following methods: 28 # 29 # setDefaults 30 # 31 # currentlyBuilding 32 # currentlyIdle 33 # currentlyInterlocked 34 # currentlyOffline 35 # currentlyWaiting 36 # 37 # setCurrentActivity 38 # updateCurrentActivity 39 # addFileToCurrentActivity 40 # finishCurrentActivity 41 # 42 # startBuild 43 # finishBuild 44 45 STDOUT = interfaces.LOG_CHANNEL_STDOUT 46 STDERR = interfaces.LOG_CHANNEL_STDERR 47 HEADER = interfaces.LOG_CHANNEL_HEADER 48 ChunkTypes = ["stdout", "stderr", "header"] 49

50 -class LogFileScanner(basic.NetstringReceiver):

51 - def __init__(self, chunk_cb, channels=[]):

52 self.chunk_cb = chunk_cb 53 self.channels = channels

54

55 - def stringReceived(self, line):

56 channel = int(line[0]) 57 if not self.channels or (channel in self.channels): 58 self.chunk_cb((channel, line[1:]))

59

60 -class LogFileProducer:

61 """What's the plan? 62 63 the LogFile has just one FD, used for both reading and writing. 64 Each time you add an entry, fd.seek to the end and then write. 65 66 Each reader (i.e. Producer) keeps track of their own offset. The reader 67 starts by seeking to the start of the logfile, and reading forwards. 68 Between each hunk of file they yield chunks, so they must remember their 69 offset before yielding and re-seek back to that offset before reading 70 more data. When their read() returns EOF, they're finished with the first 71 phase of the reading (everything that's already been written to disk). 72 73 After EOF, the remaining data is entirely in the current entries list. 74 These entries are all of the same channel, so we can do one "".join and 75 obtain a single chunk to be sent to the listener. But since that involves 76 a yield, and more data might arrive after we give up control, we have to 77 subscribe them before yielding. We can't subscribe them any earlier, 78 otherwise they'd get data out of order. 79 80 We're using a generator in the first place so that the listener can 81 throttle us, which means they're pulling. But the subscription means 82 we're pushing. Really we're a Producer. In the first phase we can be 83 either a PullProducer or a PushProducer. In the second phase we're only a 84 PushProducer. 85 86 So the client gives a LogFileConsumer to File.subscribeConsumer . This 87 Consumer must have registerProducer(), unregisterProducer(), and 88 writeChunk(), and is just like a regular twisted.interfaces.IConsumer, 89 except that writeChunk() takes chunks (tuples of (channel,text)) instead 90 of the normal write() which takes just text. The LogFileConsumer is 91 allowed to call stopProducing, pauseProducing, and resumeProducing on the 92 producer instance it is given. """ 93 94 paused = False 95 subscribed = False 96 BUFFERSIZE = 2048 97

98 - def __init__(self, logfile, consumer):

99 self.logfile = logfile 100 self.consumer = consumer 101 self.chunkGenerator = self.getChunks() 102 consumer.registerProducer(self, True)

103

104 - def getChunks(self):

105 f = self.logfile.getFile() 106 offset = 0 107 chunks = [] 108 p = LogFileScanner(chunks.append) 109 f.seek(offset) 110 data = f.read(self.BUFFERSIZE) 111 offset = f.tell() 112 while data: 113 p.dataReceived(data) 114 while chunks: 115 c = chunks.pop(0) 116 yield c 117 f.seek(offset) 118 data = f.read(self.BUFFERSIZE) 119 offset = f.tell() 120 del f 121 122 # now subscribe them to receive new entries 123 self.subscribed = True 124 self.logfile.watchers.append(self) 125 d = self.logfile.waitUntilFinished() 126 127 # then give them the not-yet-merged data 128 if self.logfile.runEntries: 129 channel = self.logfile.runEntries[0][0] 130 text = "".join([c[1] for c in self.logfile.runEntries]) 131 yield (channel, text) 132 133 # now we've caught up to the present. Anything further will come from 134 # the logfile subscription. We add the callback *after* yielding the 135 # data from runEntries, because the logfile might have finished 136 # during the yield. 137 d.addCallback(self.logfileFinished)

138

139 - def stopProducing(self):

140 # TODO: should we still call consumer.finish? probably not. 141 self.paused = True 142 self.consumer = None 143 self.done()

144

145 - def done(self):

146 if self.chunkGenerator: 147 self.chunkGenerator = None # stop making chunks 148 if self.subscribed: 149 self.logfile.watchers.remove(self) 150 self.subscribed = False

151

152 - def pauseProducing(self):

153 self.paused = True

154

155 - def resumeProducing(self):

156 # Twisted-1.3.0 has a bug which causes hangs when resumeProducing 157 # calls transport.write (there is a recursive loop, fixed in 2.0 in 158 # t.i.abstract.FileDescriptor.doWrite by setting the producerPaused 159 # flag *before* calling resumeProducing). To work around this, we 160 # just put off the real resumeProducing for a moment. This probably 161 # has a performance hit, but I'm going to assume that the log files 162 # are not retrieved frequently enough for it to be an issue. 163 164 eventually(self._resumeProducing)

165

166 - def _resumeProducing(self):

167 self.paused = False 168 if not self.chunkGenerator: 169 return 170 try: 171 while not self.paused: 172 chunk = self.chunkGenerator.next() 173 self.consumer.writeChunk(chunk) 174 # we exit this when the consumer says to stop, or we run out 175 # of chunks 176 except StopIteration: 177 # if the generator finished, it will have done releaseFile 178 self.chunkGenerator = None

179 # now everything goes through the subscription, and they don't get to 180 # pause anymore 181

182 - def logChunk(self, build, step, logfile, channel, chunk):

183 if self.consumer: 184 self.consumer.writeChunk((channel, chunk))

185

186 - def logfileFinished(self, logfile):

187 self.done() 188 if self.consumer: 189 self.consumer.unregisterProducer() 190 self.consumer.finish() 191 self.consumer = None

192

193 -def _tryremove(filename, timeout, retries):

194 """Try to remove a file, and if failed, try again in timeout. 195 Increases the timeout by a factor of 4, and only keeps trying for 196 another retries-amount of times. 197 198 """ 199 try: 200 os.unlink(filename) 201 except OSError: 202 if retries > 0: 203 reactor.callLater(timeout, _tryremove, filename, timeout * 4, 204 retries - 1) 205 else: 206 log.msg("giving up on removing %s after over %d seconds" % 207 (filename, timeout))

208

209 -class LogFile:

210 """A LogFile keeps all of its contents on disk, in a non-pickle format to 211 which new entries can easily be appended. The file on disk has a name 212 like 12-log-compile-output, under the Builder's directory. The actual 213 filename is generated (before the LogFile is created) by 214 L{BuildStatus.generateLogfileName}. 215 216 Old LogFile pickles (which kept their contents in .entries) must be 217 upgraded. The L{BuilderStatus} is responsible for doing this, when it 218 loads the L{BuildStatus} into memory. The Build pickle is not modified, 219 so users who go from 0.6.5 back to 0.6.4 don't have to lose their 220 logs.""" 221 222 implements(interfaces.IStatusLog, interfaces.ILogFile) 223 224 finished = False 225 length = 0 226 nonHeaderLength = 0 227 tailLength = 0 228 chunkSize = 10*1000 229 runLength = 0 230 # No max size by default 231 logMaxSize = None 232 # Don't keep a tail buffer by default 233 logMaxTailSize = None 234 maxLengthExceeded = False 235 runEntries = [] # provided so old pickled builds will getChunks() ok 236 entries = None 237 BUFFERSIZE = 2048 238 filename = None # relative to the Builder's basedir 239 openfile = None 240 compressMethod = "bz2" 241

242 - def __init__(self, parent, name, logfilename):

243 """ 244 @type parent: L{BuildStepStatus} 245 @param parent: the Step that this log is a part of 246 @type name: string 247 @param name: the name of this log, typically 'output' 248 @type logfilename: string 249 @param logfilename: the Builder-relative pathname for the saved entries 250 """ 251 self.step = parent 252 self.name = name 253 self.filename = logfilename 254 fn = self.getFilename() 255 if os.path.exists(fn): 256 # the buildmaster was probably stopped abruptly, before the 257 # BuilderStatus could be saved, so BuilderStatus.nextBuildNumber 258 # is out of date, and we're overlapping with earlier builds now. 259 # Warn about it, but then overwrite the old pickle file 260 log.msg("Warning: Overwriting old serialized Build at %s" % fn) 261 dirname = os.path.dirname(fn) 262 if not os.path.exists(dirname): 263 os.makedirs(dirname) 264 self.openfile = open(fn, "w+") 265 self.runEntries = [] 266 self.watchers = [] 267 self.finishedWatchers = [] 268 self.tailBuffer = []

269

270 - def getFilename(self):

271 return os.path.join(self.step.build.builder.basedir, self.filename)

272

273 - def hasContents(self):

274 return os.path.exists(self.getFilename() + '.bz2') or \ 275 os.path.exists(self.getFilename() + '.gz') or \ 276 os.path.exists(self.getFilename())

277

278 - def getName(self):

279 return self.name

280

281 - def getStep(self):

282 return self.step

283

284 - def isFinished(self):

285 return self.finished

286 - def waitUntilFinished(self):

287 if self.finished: 288 d = defer.succeed(self) 289 else: 290 d = defer.Deferred() 291 self.finishedWatchers.append(d) 292 return d

293

294 - def getFile(self):

295 if self.openfile: 296 # this is the filehandle we're using to write to the log, so 297 # don't close it! 298 return self.openfile 299 # otherwise they get their own read-only handle 300 # try a compressed log first 301 try: 302 return BZ2File(self.getFilename() + ".bz2", "r") 303 except IOError: 304 pass 305 try: 306 return GzipFile(self.getFilename() + ".gz", "r") 307 except IOError: 308 pass 309 return open(self.getFilename(), "r")

310

311 - def getText(self):

312 # this produces one ginormous string 313 return "".join(self.getChunks([STDOUT, STDERR], onlyText=True))

314

315 - def getTextWithHeaders(self):

316 return "".join(self.getChunks(onlyText=True))

317

318 - def getChunks(self, channels=[], onlyText=False):

319 # generate chunks for everything that was logged at the time we were 320 # first called, so remember how long the file was when we started. 321 # Don't read beyond that point. The current contents of 322 # self.runEntries will follow. 323 324 # this returns an iterator, which means arbitrary things could happen 325 # while we're yielding. This will faithfully deliver the log as it 326 # existed when it was started, and not return anything after that 327 # point. To use this in subscribe(catchup=True) without missing any 328 # data, you must insure that nothing will be added to the log during 329 # yield() calls. 330 331 f = self.getFile() 332 if not self.finished: 333 offset = 0 334 f.seek(0, 2) 335 remaining = f.tell() 336 else: 337 offset = 0 338 remaining = None 339 340 leftover = None 341 if self.runEntries and (not channels or 342 (self.runEntries[0][0] in channels)): 343 leftover = (self.runEntries[0][0], 344 "".join([c[1] for c in self.runEntries])) 345 346 # freeze the state of the LogFile by passing a lot of parameters into 347 # a generator 348 return self._generateChunks(f, offset, remaining, leftover, 349 channels, onlyText)

350

351 - def _generateChunks(self, f, offset, remaining, leftover, 352 channels, onlyText):

353 chunks = [] 354 p = LogFileScanner(chunks.append, channels) 355 f.seek(offset) 356 if remaining is not None: 357 data = f.read(min(remaining, self.BUFFERSIZE)) 358 remaining -= len(data) 359 else: 360 data = f.read(self.BUFFERSIZE) 361 362 offset = f.tell() 363 while data: 364 p.dataReceived(data) 365 while chunks: 366 channel, text = chunks.pop(0) 367 if onlyText: 368 yield text 369 else: 370 yield (channel, text) 371 f.seek(offset) 372 if remaining is not None: 373 data = f.read(min(remaining, self.BUFFERSIZE)) 374 remaining -= len(data) 375 else: 376 data = f.read(self.BUFFERSIZE) 377 offset = f.tell() 378 del f 379 380 if leftover: 381 if onlyText: 382 yield leftover[1] 383 else: 384 yield leftover

385

386 - def readlines(self, channel=STDOUT):

387 """Return an iterator that produces newline-terminated lines, 388 excluding header chunks.""" 389 # TODO: make this memory-efficient, by turning it into a generator 390 # that retrieves chunks as necessary, like a pull-driven version of 391 # twisted.protocols.basic.LineReceiver 392 alltext = "".join(self.getChunks([channel], onlyText=True)) 393 io = StringIO(alltext) 394 return io.readlines()

395 405

406 - def unsubscribe(self, receiver):

407 if receiver in self.watchers: 408 self.watchers.remove(receiver)

409

410 - def subscribeConsumer(self, consumer):

411 p = LogFileProducer(self, consumer) 412 p.resumeProducing()

413 414 # interface used by the build steps to add things to the log 415

416 - def merge(self):

417 # merge all .runEntries (which are all of the same type) into a 418 # single chunk for .entries 419 if not self.runEntries: 420 return 421 channel = self.runEntries[0][0] 422 text = "".join([c[1] for c in self.runEntries]) 423 assert channel < 10 424 f = self.openfile 425 f.seek(0, 2) 426 offset = 0 427 while offset < len(text): 428 size = min(len(text)-offset, self.chunkSize) 429 f.write("%d:%d" % (1 + size, channel)) 430 f.write(text[offset:offset+size]) 431 f.write(",") 432 offset += size 433 self.runEntries = [] 434 self.runLength = 0

435

436 - def addEntry(self, channel, text):

437 assert not self.finished 438 439 if isinstance(text, unicode): 440 text = text.encode('utf-8') 441 if channel != HEADER: 442 # Truncate the log if it's more than logMaxSize bytes 443 if self.logMaxSize and self.nonHeaderLength > self.logMaxSize: 444 # Add a message about what's going on 445 if not self.maxLengthExceeded: 446 msg = "\nOutput exceeded %i bytes, remaining output has been truncated\n" % self.logMaxSize 447 self.addEntry(HEADER, msg) 448 self.merge() 449 self.maxLengthExceeded = True 450 451 if self.logMaxTailSize: 452 # Update the tail buffer 453 self.tailBuffer.append((channel, text)) 454 self.tailLength += len(text) 455 while self.tailLength > self.logMaxTailSize: 456 # Drop some stuff off the beginning of the buffer 457 c,t = self.tailBuffer.pop(0) 458 n = len(t) 459 self.tailLength -= n 460 assert self.tailLength >= 0 461 return 462 463 self.nonHeaderLength += len(text) 464 465 # we only add to .runEntries here. merge() is responsible for adding 466 # merged chunks to .entries 467 if self.runEntries and channel != self.runEntries[0][0]: 468 self.merge() 469 self.runEntries.append((channel, text)) 470 self.runLength += len(text) 471 if self.runLength >= self.chunkSize: 472 self.merge() 473 474 for w in self.watchers: 475 w.logChunk(self.step.build, self.step, self, channel, text) 476 self.length += len(text)

477

478 - def addStdout(self, text):

479 self.addEntry(STDOUT, text)

480 - def addStderr(self, text):

481 self.addEntry(STDERR, text)

482 - def addHeader(self, text):

483 self.addEntry(HEADER, text)

484

485 - def finish(self):

486 if self.tailBuffer: 487 msg = "\nFinal %i bytes follow below:\n" % self.tailLength 488 tmp = self.runEntries 489 self.runEntries = [(HEADER, msg)] 490 self.merge() 491 self.runEntries = self.tailBuffer 492 self.merge() 493 self.runEntries = tmp 494 self.merge() 495 self.tailBuffer = [] 496 else: 497 self.merge() 498 499 if self.openfile: 500 # we don't do an explicit close, because there might be readers 501 # shareing the filehandle. As soon as they stop reading, the 502 # filehandle will be released and automatically closed. 503 self.openfile.flush() 504 del self.openfile 505 self.finished = True 506 watchers = self.finishedWatchers 507 self.finishedWatchers = [] 508 for w in watchers: 509 w.callback(self) 510 self.watchers = []

511 512

513 - def compressLog(self):

514 # bail out if there's no compression support 515 if self.compressMethod == "bz2": 516 compressed = self.getFilename() + ".bz2.tmp" 517 elif self.compressMethod == "gz": 518 compressed = self.getFilename() + ".gz.tmp" 519 d = threads.deferToThread(self._compressLog, compressed) 520 d.addCallback(self._renameCompressedLog, compressed) 521 d.addErrback(self._cleanupFailedCompress, compressed) 522 return d

523

524 - def _compressLog(self, compressed):

525 infile = self.getFile() 526 if self.compressMethod == "bz2": 527 cf = BZ2File(compressed, 'w') 528 elif self.compressMethod == "gz": 529 cf = GzipFile(compressed, 'w') 530 bufsize = 1024*1024 531 while True: 532 buf = infile.read(bufsize) 533 cf.write(buf) 534 if len(buf) < bufsize: 535 break 536 cf.close()

537 - def _renameCompressedLog(self, rv, compressed):

538 if self.compressMethod == "bz2": 539 filename = self.getFilename() + '.bz2' 540 else: 541 filename = self.getFilename() + '.gz' 542 if sys.platform == 'win32': 543 # windows cannot rename a file on top of an existing one, so 544 # fall back to delete-first. There are ways this can fail and 545 # lose the builder's history, so we avoid using it in the 546 # general (non-windows) case 547 if os.path.exists(filename): 548 os.unlink(filename) 549 os.rename(compressed, filename) 550 _tryremove(self.getFilename(), 1, 5)

551 - def _cleanupFailedCompress(self, failure, compressed):

552 log.msg("failed to compress %s" % self.getFilename()) 553 if os.path.exists(compressed): 554 _tryremove(compressed, 1, 5) 555 failure.trap() # reraise the failure

556 557 # persistence stuff

558 - def __getstate__(self):

559 d = self.__dict__.copy() 560 del d['step'] # filled in upon unpickling 561 del d['watchers'] 562 del d['finishedWatchers'] 563 d['entries'] = [] # let 0.6.4 tolerate the saved log. TODO: really? 564 if d.has_key('finished'): 565 del d['finished'] 566 if d.has_key('openfile'): 567 del d['openfile'] 568 return d

569

570 - def __setstate__(self, d):

571 self.__dict__ = d 572 self.watchers = [] # probably not necessary 573 self.finishedWatchers = [] # same 574 # self.step must be filled in by our parent 575 self.finished = True

576

577 - def upgrade(self, logfilename):

578 """Save our .entries to a new-style offline log file (if necessary), 579 and modify our in-memory representation to use it. The original 580 pickled LogFile (inside the pickled Build) won't be modified.""" 581 self.filename = logfilename 582 if not os.path.exists(self.getFilename()): 583 self.openfile = open(self.getFilename(), "w") 584 self.finished = False 585 for channel,text in self.entries: 586 self.addEntry(channel, text) 587 self.finish() # releases self.openfile, which will be closed 588 del self.entries

589

590 -class HTMLLogFile:

591 implements(interfaces.IStatusLog) 592 593 filename = None 594

595 - def __init__(self, parent, name, logfilename, html):

596 self.step = parent 597 self.name = name 598 self.filename = logfilename 599 self.html = html

600

601 - def getName(self):

602 return self.name # set in BuildStepStatus.addLog

603 - def getStep(self):

604 return self.step

605

606 - def isFinished(self):

607 return True

608 - def waitUntilFinished(self):

609 return defer.succeed(self)

610

611 - def hasContents(self):

612 return True

613 - def getText(self):

614 return self.html # looks kinda like text

615 - def getTextWithHeaders(self):

616 return self.html

617 - def getChunks(self):

618 return [(STDERR, self.html)]

619

622 - def unsubscribe(self, receiver):

623 pass

624

625 - def finish(self):

626 pass

627

628 - def __getstate__(self):

629 d = self.__dict__.copy() 630 del d['step'] 631 return d

632

633 - def upgrade(self, logfilename):

634 pass

635 636

637 -class Event:

638 implements(interfaces.IStatusEvent) 639 640 started = None 641 finished = None 642 text = [] 643 644 # IStatusEvent methods

645 - def getTimes(self):

646 return (self.started, self.finished)

647 - def getText(self):

648 return self.text

649 - def getLogs(self):

650 return []

651

652 - def finish(self):

653 self.finished = util.now()

654

655 -class TestResult:

656 implements(interfaces.ITestResult) 657

658 - def __init__(self, name, results, text, logs):

659 assert isinstance(name, tuple) 660 self.name = name 661 self.results = results 662 self.text = text 663 self.logs = logs

664

665 - def getName(self):

666 return self.name

667

668 - def getResults(self):

669 return self.results

670

671 - def getText(self):

672 return self.text

673

674 - def getLogs(self):

675 return self.logs

676 677

678 -class BuildSetStatus:

679 implements(interfaces.IBuildSetStatus) 680

681 - def __init__(self, bsid, status, db):

682 self.id = bsid 683 self.status = status 684 self.db = db

685

686 - def _get_info(self):

687 return self.db.get_buildset_info(self.id)

688 689 # methods for our clients 690

691 - def getSourceStamp(self):

692 (external_idstring, reason, ssid, complete, results) = self._get_info() 693 return self.db.getSourceStampNumberedNow(ssid)

694

695 - def getReason(self):

696 (external_idstring, reason, ssid, complete, results) = self._get_info() 697 return reason

698 - def getResults(self):

699 (external_idstring, reason, ssid, complete, results) = self._get_info() 700 return results

701 - def getID(self):

702 (external_idstring, reason, ssid, complete, results) = self._get_info() 703 return external_idstring

704

705 - def getBuilderNamesAndBuildRequests(self):

706 brs = {} 707 brids = self.db.get_buildrequestids_for_buildset(self.id) 708 for (buildername, brid) in brids.items(): 709 brs[buildername] = BuildRequestStatus(brid, self.status, self.db) 710 return brs

711

712 - def getBuilderNames(self):

713 brs = self.db.get_buildrequestids_for_buildset(self.id) 714 return sorted(brs.keys())

715

716 - def getBuildRequests(self):

717 brs = self.db.get_buildrequestids_for_buildset(self.id) 718 return [BuildRequestStatus(brid, self.status, self.db) 719 for brid in brs.values()]

720

721 - def isFinished(self):

722 (external_idstring, reason, ssid, complete, results) = self._get_info() 723 return complete

724

725 - def waitUntilSuccess(self):

726 return self.status._buildset_waitUntilSuccess(self.id)

727 - def waitUntilFinished(self):

728 return self.status._buildset_waitUntilFinished(self.id)

729

730 -class BuildRequestStatus:

731 implements(interfaces.IBuildRequestStatus) 732

733 - def __init__(self, brid, status, db):

734 self.brid = brid 735 self.status = status 736 self.db = db

737

738 - def buildStarted(self, build):

739 self.status._buildrequest_buildStarted(build.status) 740 self.builds.append(build.status)

741 742 # methods called by our clients

743 - def getSourceStamp(self):

744 br = self.db.getBuildRequestWithNumber(self.brid) 745 return br.source

746 - def getBuilderName(self):

747 br = self.db.getBuildRequestWithNumber(self.brid) 748 return br.builderName

749 - def getBuilds(self):

750 builder = self.status.getBuilder(self.getBuilderName()) 751 builds = [] 752 buildnums = sorted(self.db.get_buildnums_for_brid(self.brid)) 753 for buildnum in buildnums: 754 bs = builder.getBuild(buildnum) 755 if bs: 756 builds.append(bs) 757 return builds

758

764 - def unsubscribe(self, observer):

765 self.status._buildrequest_unsubscribe(self.brid, observer)

766

767 - def getSubmitTime(self):

768 br = self.db.getBuildRequestWithNumber(self.brid) 769 return br.submittedAt

770

771 - def asDict(self):

772 result = {} 773 # Constant 774 result['source'] = self.getSourceStamp().asDict() 775 result['builderName'] = self.getBuilderName() 776 result['submittedAt'] = self.getSubmitTime() 777 778 # Transient 779 result['builds'] = [build.asDict() for build in self.getBuilds()] 780 return result

781 782

783 -class BuildStepStatus(styles.Versioned):

784 """ 785 I represent a collection of output status for a 786 L{buildbot.process.step.BuildStep}. 787 788 Statistics contain any information gleaned from a step that is 789 not in the form of a logfile. As an example, steps that run 790 tests might gather statistics about the number of passed, failed, 791 or skipped tests. 792 793 @type progress: L{buildbot.status.progress.StepProgress} 794 @cvar progress: tracks ETA for the step 795 @type text: list of strings 796 @cvar text: list of short texts that describe the command and its status 797 @type text2: list of strings 798 @cvar text2: list of short texts added to the overall build description 799 @type logs: dict of string -> L{buildbot.status.builder.LogFile} 800 @ivar logs: logs of steps 801 @type statistics: dict 802 @ivar statistics: results from running this step 803 """ 804 # note that these are created when the Build is set up, before each 805 # corresponding BuildStep has started. 806 implements(interfaces.IBuildStepStatus, interfaces.IStatusEvent) 807 persistenceVersion = 2 808 809 started = None 810 finished = None 811 progress = None 812 text = [] 813 results = (None, []) 814 text2 = [] 815 watchers = [] 816 updates = {} 817 finishedWatchers = [] 818 statistics = {} 819

820 - def __init__(self, parent):

821 assert interfaces.IBuildStatus(parent) 822 self.build = parent 823 self.logs = [] 824 self.urls = {} 825 self.watchers = [] 826 self.updates = {} 827 self.finishedWatchers = [] 828 self.statistics = {}

829

830 - def getName(self):

831 """Returns a short string with the name of this step. This string 832 may have spaces in it.""" 833 return self.name

834

835 - def getBuild(self):

836 return self.build

837

838 - def getTimes(self):

839 return (self.started, self.finished)

840

841 - def getExpectations(self):

842 """Returns a list of tuples (name, current, target).""" 843 if not self.progress: 844 return [] 845 ret = [] 846 metrics = self.progress.progress.keys() 847 metrics.sort() 848 for m in metrics: 849 t = (m, self.progress.progress[m], self.progress.expectations[m]) 850 ret.append(t) 851 return ret

852

853 - def getLogs(self):

854 return self.logs

855

856 - def getURLs(self):

857 return self.urls.copy()

858

859 - def isStarted(self):

860 return (self.started is not None)

861

862 - def isFinished(self):

863 return (self.finished is not None)

864

865 - def waitUntilFinished(self):

866 if self.finished: 867 d = defer.succeed(self) 868 else: 869 d = defer.Deferred() 870 self.finishedWatchers.append(d) 871 return d

872 873 # while the step is running, the following methods make sense. 874 # Afterwards they return None 875

876 - def getETA(self):

877 if self.started is None: 878 return None # not started yet 879 if self.finished is not None: 880 return None # already finished 881 if not self.progress: 882 return None # no way to predict 883 return self.progress.remaining()

884 885 # Once you know the step has finished, the following methods are legal. 886 # Before this step has finished, they all return None. 887

888 - def getText(self):

889 """Returns a list of strings which describe the step. These are 890 intended to be displayed in a narrow column. If more space is 891 available, the caller should join them together with spaces before 892 presenting them to the user.""" 893 return self.text

894

895 - def getResults(self):

896 """Return a tuple describing the results of the step. 897 'result' is one of the constants in L{buildbot.status.builder}: 898 SUCCESS, WARNINGS, FAILURE, or SKIPPED. 899 'strings' is an optional list of strings that the step wants to 900 append to the overall build's results. These strings are usually 901 more terse than the ones returned by getText(): in particular, 902 successful Steps do not usually contribute any text to the 903 overall build. 904 905 @rtype: tuple of int, list of strings 906 @returns: (result, strings) 907 """ 908 return (self.results, self.text2)

909

910 - def hasStatistic(self, name):

911 """Return true if this step has a value for the given statistic. 912 """ 913 return self.statistics.has_key(name)

914

915 - def getStatistic(self, name, default=None):

916 """Return the given statistic, if present 917 """ 918 return self.statistics.get(name, default)

919 920 # subscription interface 921 927

928 - def sendETAUpdate(self, receiver, updateInterval):

929 self.updates[receiver] = None 930 # they might unsubscribe during stepETAUpdate 931 receiver.stepETAUpdate(self.build, self, 932 self.getETA(), self.getExpectations()) 933 if receiver in self.watchers: 934 self.updates[receiver] = reactor.callLater(updateInterval, 935 self.sendETAUpdate, 936 receiver, 937 updateInterval)

938

939 - def unsubscribe(self, receiver):

940 if receiver in self.watchers: 941 self.watchers.remove(receiver) 942 if receiver in self.updates: 943 if self.updates[receiver] is not None: 944 self.updates[receiver].cancel() 945 del self.updates[receiver]

946 947 948 # methods to be invoked by the BuildStep 949

950 - def setName(self, stepname):

951 self.name = stepname

952

953 - def setColor(self, color):

954 log.msg("BuildStepStatus.setColor is no longer supported -- ignoring color %s" % (color,))

955

956 - def setProgress(self, stepprogress):

957 self.progress = stepprogress

958

959 - def stepStarted(self):

960 self.started = util.now() 961 if self.build: 962 self.build.stepStarted(self)

963

964 - def addLog(self, name):

965 assert self.started # addLog before stepStarted won't notify watchers 966 logfilename = self.build.generateLogfileName(self.name, name) 967 log = LogFile(self, name, logfilename) 968 log.logMaxSize = self.build.builder.logMaxSize 969 log.logMaxTailSize = self.build.builder.logMaxTailSize 970 log.compressMethod = self.build.builder.logCompressionMethod 971 self.logs.append(log) 972 for w in self.watchers: 973 receiver = w.logStarted(self.build, self, log) 974 if receiver: 975 log.subscribe(receiver, True) 976 d = log.waitUntilFinished() 977 d.addCallback(lambda log: log.unsubscribe(receiver)) 978 d = log.waitUntilFinished() 979 d.addCallback(self.logFinished) 980 return log

981

982 - def addHTMLLog(self, name, html):

983 assert self.started # addLog before stepStarted won't notify watchers 984 logfilename = self.build.generateLogfileName(self.name, name) 985 log = HTMLLogFile(self, name, logfilename, html) 986 self.logs.append(log) 987 for w in self.watchers: 988 receiver = w.logStarted(self.build, self, log) 989 # TODO: think about this: there isn't much point in letting 990 # them subscribe 991 #if receiver: 992 # log.subscribe(receiver, True) 993 w.logFinished(self.build, self, log)

994

995 - def logFinished(self, log):

996 for w in self.watchers: 997 w.logFinished(self.build, self, log)

998

999 - def addURL(self, name, url):

1000 self.urls[name] = url

1001

1002 - def setText(self, text):

1003 self.text = text 1004 for w in self.watchers: 1005 w.stepTextChanged(self.build, self, text)

1006 - def setText2(self, text):

1007 self.text2 = text 1008 for w in self.watchers: 1009 w.stepText2Changed(self.build, self, text)

1010

1011 - def setStatistic(self, name, value):

1012 """Set the given statistic. Usually called by subclasses. 1013 """ 1014 self.statistics[name] = value

1015

1016 - def stepFinished(self, results):

1017 self.finished = util.now() 1018 self.results = results 1019 cld = [] # deferreds for log compression 1020 logCompressionLimit = self.build.builder.logCompressionLimit 1021 for loog in self.logs: 1022 if not loog.isFinished(): 1023 loog.finish() 1024 # if log compression is on, and it's a real LogFile, 1025 # HTMLLogFiles aren't files 1026 if logCompressionLimit is not False and \ 1027 isinstance(loog, LogFile): 1028 if os.path.getsize(loog.getFilename()) > logCompressionLimit: 1029 loog_deferred = loog.compressLog() 1030 if loog_deferred: 1031 cld.append(loog_deferred) 1032 1033 for r in self.updates.keys(): 1034 if self.updates[r] is not None: 1035 self.updates[r].cancel() 1036 del self.updates[r] 1037 1038 watchers = self.finishedWatchers 1039 self.finishedWatchers = [] 1040 for w in watchers: 1041 w.callback(self) 1042 if cld: 1043 return defer.DeferredList(cld)

1044

1045 - def checkLogfiles(self):

1046 # filter out logs that have been deleted 1047 self.logs = [ l for l in self.logs if l.hasContents() ]

1048 1049 # persistence 1050

1051 - def __getstate__(self):

1052 d = styles.Versioned.__getstate__(self) 1053 del d['build'] # filled in when loading 1054 if d.has_key('progress'): 1055 del d['progress'] 1056 del d['watchers'] 1057 del d['finishedWatchers'] 1058 del d['updates'] 1059 return d

1060

1061 - def __setstate__(self, d):

1062 styles.Versioned.__setstate__(self, d) 1063 # self.build must be filled in by our parent 1064 1065 # point the logs to this object 1066 for loog in self.logs: 1067 loog.step = self

1068

1069 - def upgradeToVersion1(self):

1070 if not hasattr(self, "urls"): 1071 self.urls = {}

1072

1073 - def upgradeToVersion2(self):

1074 if not hasattr(self, "statistics"): 1075 self.statistics = {}

1076

1077 - def asDict(self):

1078 result = {} 1079 # Constant 1080 result['name'] = self.getName() 1081 1082 # Transient 1083 result['text'] = self.getText() 1084 result['results'] = self.getResults() 1085 result['isStarted'] = self.isStarted() 1086 result['isFinished'] = self.isFinished() 1087 result['statistics'] = self.statistics 1088 result['times'] = self.getTimes() 1089 result['expectations'] = self.getExpectations() 1090 result['eta'] = self.getETA() 1091 result['urls'] = self.getURLs() 1092 # TODO(maruel): Move that to a sub-url or just publish the log_url 1093 # instead. 1094 #result['logs'] = self.getLogs() 1095 return result

1096 1097

1098 -class BuildStatus(styles.Versioned):

1099 implements(interfaces.IBuildStatus, interfaces.IStatusEvent) 1100 persistenceVersion = 3 1101 1102 source = None 1103 reason = None 1104 changes = [] 1105 blamelist = [] 1106 progress = None 1107 started = None 1108 finished = None 1109 currentStep = None 1110 text = [] 1111 results = None 1112 slavename = "???" 1113 1114 # these lists/dicts are defined here so that unserialized instances have 1115 # (empty) values. They are set in __init__ to new objects to make sure 1116 # each instance gets its own copy. 1117 watchers = [] 1118 updates = {} 1119 finishedWatchers = [] 1120 testResults = {} 1121

1122 - def __init__(self, parent, number):

1123 """ 1124 @type parent: L{BuilderStatus} 1125 @type number: int 1126 """ 1127 assert interfaces.IBuilderStatus(parent) 1128 self.builder = parent 1129 self.number = number 1130 self.watchers = [] 1131 self.updates = {} 1132 self.finishedWatchers = [] 1133 self.steps = [] 1134 self.testResults = {} 1135 self.properties = Properties()

1136

1137 - def __repr__(self):

1138 return "<%s #%s>" % (self.__class__.__name__, self.number)

1139 1140 # IBuildStatus 1141

1142 - def getBuilder(self):

1143 """ 1144 @rtype: L{BuilderStatus} 1145 """ 1146 return self.builder

1147

1148 - def getProperty(self, propname):

1149 return self.properties[propname]

1150

1151 - def getProperties(self):

1152 return self.properties

1153

1154 - def getNumber(self):

1155 return self.number

1156

1157 - def getPreviousBuild(self):

1158 if self.number == 0: 1159 return None 1160 return self.builder.getBuild(self.number-1)

1161

1162 - def getSourceStamp(self, absolute=False):

1163 if not absolute or not self.properties.has_key('got_revision'): 1164 return self.source 1165 return self.source.getAbsoluteSourceStamp(self.properties['got_revision'])

1166

1167 - def getReason(self):

1168 return self.reason

1169

1170 - def getChanges(self):

1171 return self.changes

1172

1173 - def getResponsibleUsers(self):

1174 return self.blamelist

1175

1176 - def getInterestedUsers(self):

1177 # TODO: the Builder should add others: sheriffs, domain-owners 1178 return self.blamelist + self.properties.getProperty('owners', [])

1179

1180 - def getSteps(self):

1181 """Return a list of IBuildStepStatus objects. For invariant builds 1182 (those which always use the same set of Steps), this should be the 1183 complete list, however some of the steps may not have started yet 1184 (step.getTimes()[0] will be None). For variant builds, this may not 1185 be complete (asking again later may give you more of them).""" 1186 return self.steps

1187

1188 - def getTimes(self):

1189 return (self.started, self.finished)

1190 1191 _sentinel = [] # used as a sentinel to indicate unspecified initial_value

1192 - def getSummaryStatistic(self, name, summary_fn, initial_value=_sentinel):

1193 """Summarize the named statistic over all steps in which it 1194 exists, using combination_fn and initial_value to combine multiple 1195 results into a single result. This translates to a call to Python's 1196 X{reduce}:: 1197 return reduce(summary_fn, step_stats_list, initial_value) 1198 """ 1199 step_stats_list = [ 1200 st.getStatistic(name) 1201 for st in self.steps 1202 if st.hasStatistic(name) ] 1203 if initial_value is self._sentinel: 1204 return reduce(summary_fn, step_stats_list) 1205 else: 1206 return reduce(summary_fn, step_stats_list, initial_value)

1207

1208 - def isFinished(self):

1209 return (self.finished is not None)

1210

1211 - def waitUntilFinished(self):

1212 if self.finished: 1213 d = defer.succeed(self) 1214 else: 1215 d = defer.Deferred() 1216 self.finishedWatchers.append(d) 1217 return d

1218 1219 # while the build is running, the following methods make sense. 1220 # Afterwards they return None 1221

1222 - def getETA(self):

1223 if self.finished is not None: 1224 return None 1225 if not self.progress: 1226 return None 1227 eta = self.progress.eta() 1228 if eta is None: 1229 return None 1230 return eta - util.now()

1231

1232 - def getCurrentStep(self):

1233 return self.currentStep

1234 1235 # Once you know the build has finished, the following methods are legal. 1236 # Before ths build has finished, they all return None. 1237

1238 - def getText(self):

1239 text = [] 1240 text.extend(self.text) 1241 for s in self.steps: 1242 text.extend(s.text2) 1243 return text

1244

1245 - def getResults(self):

1246 return self.results

1247

1248 - def getSlavename(self):

1249 return self.slavename

1250

1251 - def getTestResults(self):

1252 return self.testResults

1253

1254 - def getLogs(self):

1255 # TODO: steps should contribute significant logs instead of this 1256 # hack, which returns every log from every step. The logs should get 1257 # names like "compile" and "test" instead of "compile.output" 1258 logs = [] 1259 for s in self.steps: 1260 for log in s.getLogs(): 1261 logs.append(log) 1262 return logs

1263 1264 # subscription interface 1265 1272

1273 - def sendETAUpdate(self, receiver, updateInterval):

1274 self.updates[receiver] = None 1275 ETA = self.getETA() 1276 if ETA is not None: 1277 receiver.buildETAUpdate(self, self.getETA()) 1278 # they might have unsubscribed during buildETAUpdate 1279 if receiver in self.watchers: 1280 self.updates[receiver] = reactor.callLater(updateInterval, 1281 self.sendETAUpdate, 1282 receiver, 1283 updateInterval)

1284

1285 - def unsubscribe(self, receiver):

1286 if receiver in self.watchers: 1287 self.watchers.remove(receiver) 1288 if receiver in self.updates: 1289 if self.updates[receiver] is not None: 1290 self.updates[receiver].cancel() 1291 del self.updates[receiver]

1292 1293 # methods for the base.Build to invoke 1294

1295 - def addStepWithName(self, name):

1296 """The Build is setting up, and has added a new BuildStep to its 1297 list. Create a BuildStepStatus object to which it can send status 1298 updates.""" 1299 1300 s = BuildStepStatus(self) 1301 s.setName(name) 1302 self.steps.append(s) 1303 return s

1304

1305 - def setProperty(self, propname, value, source):

1306 self.properties.setProperty(propname, value, source)

1307

1308 - def addTestResult(self, result):

1309 self.testResults[result.getName()] = result

1310

1311 - def setSourceStamp(self, sourceStamp):

1312 self.source = sourceStamp 1313 self.changes = self.source.changes

1314

1315 - def setReason(self, reason):

1316 self.reason = reason

1317 - def setBlamelist(self, blamelist):

1318 self.blamelist = blamelist

1319 - def setProgress(self, progress):

1320 self.progress = progress

1321

1322 - def buildStarted(self, build):

1323 """The Build has been set up and is about to be started. It can now 1324 be safely queried, so it is time to announce the new build.""" 1325 1326 self.started = util.now() 1327 # now that we're ready to report status, let the BuilderStatus tell 1328 # the world about us 1329 self.builder.buildStarted(self)

1330

1331 - def setSlavename(self, slavename):

1332 self.slavename = slavename

1333

1334 - def setText(self, text):

1335 assert isinstance(text, (list, tuple)) 1336 self.text = text

1337 - def setResults(self, results):

1338 self.results = results

1339

1340 - def buildFinished(self):

1341 self.currentStep = None 1342 self.finished = util.now() 1343 1344 for r in self.updates.keys(): 1345 if self.updates[r] is not None: 1346 self.updates[r].cancel() 1347 del self.updates[r] 1348 1349 watchers = self.finishedWatchers 1350 self.finishedWatchers = [] 1351 for w in watchers: 1352 w.callback(self)

1353 1354 # methods called by our BuildStepStatus children 1355

1356 - def stepStarted(self, step):

1357 self.currentStep = step 1358 for w in self.watchers: 1359 receiver = w.stepStarted(self, step) 1360 if receiver: 1361 if type(receiver) == type(()): 1362 step.subscribe(receiver[0], receiver[1]) 1363 else: 1364 step.subscribe(receiver) 1365 d = step.waitUntilFinished() 1366 d.addCallback(lambda step: step.unsubscribe(receiver)) 1367 1368 step.waitUntilFinished().addCallback(self._stepFinished)

1369

1370 - def _stepFinished(self, step):

1371 results = step.getResults() 1372 for w in self.watchers: 1373 w.stepFinished(self, step, results)

1374 1375 # methods called by our BuilderStatus parent 1376

1377 - def pruneSteps(self):

1378 # this build is very old: remove the build steps too 1379 self.steps = []

1380 1381 # persistence stuff 1382

1383 - def generateLogfileName(self, stepname, logname):

1384 """Return a filename (relative to the Builder's base directory) where 1385 the logfile's contents can be stored uniquely. 1386 1387 The base filename is made by combining our build number, the Step's 1388 name, and the log's name, then removing unsuitable characters. The 1389 filename is then made unique by appending _0, _1, etc, until it does 1390 not collide with any other logfile. 1391 1392 These files are kept in the Builder's basedir (rather than a 1393 per-Build subdirectory) because that makes cleanup easier: cron and 1394 find will help get rid of the old logs, but the empty directories are 1395 more of a hassle to remove.""" 1396 1397 starting_filename = "%d-log-%s-%s" % (self.number, stepname, logname) 1398 starting_filename = re.sub(r'[^\w\.\-]', '_', starting_filename) 1399 # now make it unique 1400 unique_counter = 0 1401 filename = starting_filename 1402 while filename in [l.filename 1403 for step in self.steps 1404 for l in step.getLogs() 1405 if l.filename]: 1406 filename = "%s_%d" % (starting_filename, unique_counter) 1407 unique_counter += 1 1408 return filename

1409

1410 - def __getstate__(self):

1411 d = styles.Versioned.__getstate__(self) 1412 # for now, a serialized Build is always "finished". We will never 1413 # save unfinished builds. 1414 if not self.finished: 1415 d['finished'] = True 1416 # TODO: push an "interrupted" step so it is clear that the build 1417 # was interrupted. The builder will have a 'shutdown' event, but 1418 # someone looking at just this build will be confused as to why 1419 # the last log is truncated. 1420 for k in 'builder', 'watchers', 'updates', 'finishedWatchers': 1421 if k in d: del d[k] 1422 return d

1423

1424 - def __setstate__(self, d):

1425 styles.Versioned.__setstate__(self, d) 1426 # self.builder must be filled in by our parent when loading 1427 for step in self.steps: 1428 step.build = self 1429 self.watchers = [] 1430 self.updates = {} 1431 self.finishedWatchers = []

1432

1433 - def upgradeToVersion1(self):

1434 if hasattr(self, "sourceStamp"): 1435 # the old .sourceStamp attribute wasn't actually very useful 1436 maxChangeNumber, patch = self.sourceStamp 1437 changes = getattr(self, 'changes', []) 1438 source = sourcestamp.SourceStamp(branch=None, 1439 revision=None, 1440 patch=patch, 1441 changes=changes) 1442 self.source = source 1443 self.changes = source.changes 1444 del self.sourceStamp

1445

1446 - def upgradeToVersion2(self):

1447 self.properties = {}

1448

1449 - def upgradeToVersion3(self):

1450 # in version 3, self.properties became a Properties object 1451 propdict = self.properties 1452 self.properties = Properties() 1453 self.properties.update(propdict, "Upgrade from previous version")

1454

1455 - def upgradeLogfiles(self):

1456 # upgrade any LogFiles that need it. This must occur after we've been 1457 # attached to our Builder, and after we know about all LogFiles of 1458 # all Steps (to get the filenames right). 1459 assert self.builder 1460 for s in self.steps: 1461 for l in s.getLogs(): 1462 if l.filename: 1463 pass # new-style, log contents are on disk 1464 else: 1465 logfilename = self.generateLogfileName(s.name, l.name) 1466 # let the logfile update its .filename pointer, 1467 # transferring its contents onto disk if necessary 1468 l.upgrade(logfilename)

1469

1470 - def checkLogfiles(self):

1471 # check that all logfiles exist, and remove references to any that 1472 # have been deleted (e.g., by purge()) 1473 for s in self.steps: 1474 s.checkLogfiles()

1475

1476 - def saveYourself(self):

1477 filename = os.path.join(self.builder.basedir, "%d" % self.number) 1478 if os.path.isdir(filename): 1479 # leftover from 0.5.0, which stored builds in directories 1480 shutil.rmtree(filename, ignore_errors=True) 1481 tmpfilename = filename + ".tmp" 1482 try: 1483 dump(self, open(tmpfilename, "wb"), -1) 1484 if sys.platform == 'win32': 1485 # windows cannot rename a file on top of an existing one, so 1486 # fall back to delete-first. There are ways this can fail and 1487 # lose the builder's history, so we avoid using it in the 1488 # general (non-windows) case 1489 if os.path.exists(filename): 1490 os.unlink(filename) 1491 os.rename(tmpfilename, filename) 1492 except: 1493 log.msg("unable to save build %s-#%d" % (self.builder.name, 1494 self.number)) 1495 log.err()

1496

1497 - def asDict(self):

1498 result = {} 1499 # Constant 1500 result['builderName'] = self.builder.name 1501 result['number'] = self.getNumber() 1502 result['sourceStamp'] = self.getSourceStamp().asDict() 1503 result['reason'] = self.getReason() 1504 result['blame'] = self.getResponsibleUsers() 1505 1506 # Transient 1507 result['properties'] = self.getProperties().asList() 1508 result['times'] = self.getTimes() 1509 result['text'] = self.getText() 1510 result['results'] = self.getResults() 1511 result['slave'] = self.getSlavename() 1512 # TODO(maruel): Add. 1513 #result['test_results'] = self.getTestResults() 1514 # TODO(maruel): Include the url? It's too heavy otherwise. 1515 #result['logs'] = self.getLogs() 1516 result['eta'] = self.getETA() 1517 result['steps'] = [bss.asDict() for bss in self.steps] 1518 if self.getCurrentStep(): 1519 result['currentStep'] = self.getCurrentStep().asDict() 1520 else: 1521 result['currentStep'] = None 1522 return result

1523 1524 1525

1526 -class BuilderStatus(styles.Versioned):

1527 """I handle status information for a single process.base.Builder object. 1528 That object sends status changes to me (frequently as Events), and I 1529 provide them on demand to the various status recipients, like the HTML 1530 waterfall display and the live status clients. It also sends build 1531 summaries to me, which I log and provide to status clients who aren't 1532 interested in seeing details of the individual build steps. 1533 1534 I am responsible for maintaining the list of historic Events and Builds, 1535 pruning old ones, and loading them from / saving them to disk. 1536 1537 I live in the buildbot.process.base.Builder object, in the 1538 .builder_status attribute. 1539 1540 @type category: string 1541 @ivar category: user-defined category this builder belongs to; can be 1542 used to filter on in status clients 1543 """ 1544 1545 implements(interfaces.IBuilderStatus, interfaces.IEventSource) 1546 persistenceVersion = 1 1547 1548 # these limit the amount of memory we consume, as well as the size of the 1549 # main Builder pickle. The Build and LogFile pickles on disk must be 1550 # handled separately. 1551 buildCacheSize = 15 1552 eventHorizon = 50 # forget events beyond this 1553 1554 # these limit on-disk storage 1555 logHorizon = 40 # forget logs in steps in builds beyond this 1556 buildHorizon = 100 # forget builds beyond this 1557 1558 category = None 1559 currentBigState = "offline" # or idle/waiting/interlocked/building 1560 basedir = None # filled in by our parent 1561

1562 - def __init__(self, buildername, category=None):

1563 self.name = buildername 1564 self.category = category 1565 1566 self.slavenames = [] 1567 self.events = [] 1568 # these three hold Events, and are used to retrieve the current 1569 # state of the boxes. 1570 self.lastBuildStatus = None 1571 #self.currentBig = None 1572 #self.currentSmall = None 1573 self.currentBuilds = [] 1574 self.nextBuild = None 1575 self.watchers = [] 1576 self.buildCache = weakref.WeakValueDictionary() 1577 self.buildCache_LRU = [] 1578 self.logCompressionLimit = False # default to no compression for tests 1579 self.logCompressionMethod = "bz2" 1580 self.logMaxSize = None # No default limit 1581 self.logMaxTailSize = None # No tail buffering

1582 1583 # persistence 1584

1585 - def __getstate__(self):

1586 # when saving, don't record transient stuff like what builds are 1587 # currently running, because they won't be there when we start back 1588 # up. Nor do we save self.watchers, nor anything that gets set by our 1589 # parent like .basedir and .status 1590 d = styles.Versioned.__getstate__(self) 1591 d['watchers'] = [] 1592 del d['buildCache'] 1593 del d['buildCache_LRU'] 1594 for b in self.currentBuilds: 1595 b.saveYourself() 1596 # TODO: push a 'hey, build was interrupted' event 1597 del d['currentBuilds'] 1598 d.pop('pendingBuilds', None) 1599 del d['currentBigState'] 1600 del d['basedir'] 1601 del d['status'] 1602 del d['nextBuildNumber'] 1603 return d

1604

1605 - def __setstate__(self, d):

1606 # when loading, re-initialize the transient stuff. Remember that 1607 # upgradeToVersion1 and such will be called after this finishes. 1608 styles.Versioned.__setstate__(self, d) 1609 self.buildCache = weakref.WeakValueDictionary() 1610 self.buildCache_LRU = [] 1611 self.currentBuilds = [] 1612 self.watchers = [] 1613 self.slavenames = []

1614 # self.basedir must be filled in by our parent 1615 # self.status must be filled in by our parent 1616

1617 - def reconfigFromBuildmaster(self, buildmaster):

1618 # Note that we do not hang onto the buildmaster, since this object 1619 # gets pickled and unpickled. 1620 if buildmaster.buildCacheSize: 1621 self.buildCacheSize = buildmaster.buildCacheSize

1622

1623 - def upgradeToVersion1(self):

1624 if hasattr(self, 'slavename'): 1625 self.slavenames = [self.slavename] 1626 del self.slavename 1627 if hasattr(self, 'nextBuildNumber'): 1628 del self.nextBuildNumber # determineNextBuildNumber chooses this

1629

1630 - def determineNextBuildNumber(self):

1631 """Scan our directory of saved BuildStatus instances to determine 1632 what our self.nextBuildNumber should be. Set it one larger than the 1633 highest-numbered build we discover. This is called by the top-level 1634 Status object shortly after we are created or loaded from disk. 1635 """ 1636 existing_builds = [int(f) 1637 for f in os.listdir(self.basedir) 1638 if re.match("^\d+$", f)] 1639 if existing_builds: 1640 self.nextBuildNumber = max(existing_builds) + 1 1641 else: 1642 self.nextBuildNumber = 0

1643

1644 - def setLogCompressionLimit(self, lowerLimit):

1645 self.logCompressionLimit = lowerLimit

1646

1647 - def setLogCompressionMethod(self, method):

1648 assert method in ("bz2", "gz") 1649 self.logCompressionMethod = method

1650

1651 - def setLogMaxSize(self, upperLimit):

1652 self.logMaxSize = upperLimit

1653

1654 - def setLogMaxTailSize(self, tailSize):

1655 self.logMaxTailSize = tailSize

1656

1657 - def saveYourself(self):

1658 for b in self.currentBuilds: 1659 if not b.isFinished: 1660 # interrupted build, need to save it anyway. 1661 # BuildStatus.saveYourself will mark it as interrupted. 1662 b.saveYourself() 1663 filename = os.path.join(self.basedir, "builder") 1664 tmpfilename = filename + ".tmp" 1665 try: 1666 dump(self, open(tmpfilename, "wb"), -1) 1667 if sys.platform == 'win32': 1668 # windows cannot rename a file on top of an existing one 1669 if os.path.exists(filename): 1670 os.unlink(filename) 1671 os.rename(tmpfilename, filename) 1672 except: 1673 log.msg("unable to save builder %s" % self.name) 1674 log.err()

1675 1676 1677 # build cache management 1678

1679 - def makeBuildFilename(self, number):

1680 return os.path.join(self.basedir, "%d" % number)

1681

1682 - def touchBuildCache(self, build):

1683 self.buildCache[build.number] = build 1684 if build in self.buildCache_LRU: 1685 self.buildCache_LRU.remove(build) 1686 self.buildCache_LRU = self.buildCache_LRU[-(self.buildCacheSize-1):] + [ build ] 1687 return build

1688

1689 - def getBuildByNumber(self, number):

1690 # first look in currentBuilds 1691 for b in self.currentBuilds: 1692 if b.number == number: 1693 return self.touchBuildCache(b) 1694 1695 # then in the buildCache 1696 if number in self.buildCache: 1697 return self.touchBuildCache(self.buildCache[number]) 1698 1699 # then fall back to loading it from disk 1700 filename = self.makeBuildFilename(number) 1701 try: 1702 log.msg("Loading builder %s's build %d from on-disk pickle" 1703 % (self.name, number)) 1704 build = load(open(filename, "rb")) 1705 styles.doUpgrade() 1706 build.builder = self 1707 # handle LogFiles from after 0.5.0 and before 0.6.5 1708 build.upgradeLogfiles() 1709 # check that logfiles exist 1710 build.checkLogfiles() 1711 return self.touchBuildCache(build) 1712 except IOError: 1713 raise IndexError("no such build %d" % number) 1714 except EOFError: 1715 raise IndexError("corrupted build pickle %d" % number)

1716

1717 - def prune(self):

1718 gc.collect() 1719 1720 # begin by pruning our own events 1721 self.events = self.events[-self.eventHorizon:] 1722 1723 # get the horizons straight 1724 if self.buildHorizon: 1725 earliest_build = self.nextBuildNumber - self.buildHorizon 1726 else: 1727 earliest_build = 0 1728 1729 if self.logHorizon: 1730 earliest_log = self.nextBuildNumber - self.logHorizon 1731 else: 1732 earliest_log = 0 1733 1734 if earliest_log < earliest_build: 1735 earliest_log = earliest_build 1736 1737 if earliest_build == 0: 1738 return 1739 1740 # skim the directory and delete anything that shouldn't be there anymore 1741 build_re = re.compile(r"^([0-9]+)$") 1742 build_log_re = re.compile(r"^([0-9]+)-.*$") 1743 # if the directory doesn't exist, bail out here 1744 if not os.path.exists(self.basedir): 1745 return 1746 1747 for filename in os.listdir(self.basedir): 1748 num = None 1749 mo = build_re.match(filename) 1750 is_logfile = False 1751 if mo: 1752 num = int(mo.group(1)) 1753 else: 1754 mo = build_log_re.match(filename) 1755 if mo: 1756 num = int(mo.group(1)) 1757 is_logfile = True 1758 1759 if num is None: continue 1760 if num in self.buildCache: continue 1761 1762 if (is_logfile and num < earliest_log) or num < earliest_build: 1763 pathname = os.path.join(self.basedir, filename) 1764 log.msg("pruning '%s'" % pathname) 1765 try: os.unlink(pathname) 1766 except OSError: pass

1767 1768 # IBuilderStatus methods

1769 - def getName(self):

1770 return self.name

1771

1772 - def getState(self):

1773 return (self.currentBigState, self.currentBuilds)

1774

1775 - def getSlaves(self):

1776 return [self.status.getSlave(name) for name in self.slavenames]

1777

1778 - def getPendingBuilds(self):

1779 db = self.status.db 1780 return [BuildRequestStatus(brid, self.status, db) 1781 for brid in db.get_pending_brids_for_builder(self.name)]

1782

1783 - def getCurrentBuilds(self):

1784 return self.currentBuilds

1785

1786 - def getLastFinishedBuild(self):

1787 b = self.getBuild(-1) 1788 if not (b and b.isFinished()): 1789 b = self.getBuild(-2) 1790 return b

1791

1792 - def getCategory(self):

1793 return self.category

1794

1795 - def getBuild(self, number):

1796 if number < 0: 1797 number = self.nextBuildNumber + number 1798 if number < 0 or number >= self.nextBuildNumber: 1799 return None 1800 1801 try: 1802 return self.getBuildByNumber(number) 1803 except IndexError: 1804 return None

1805

1806 - def getEvent(self, number):

1807 try: 1808 return self.events[number] 1809 except IndexError: 1810 return None

1811

1812 - def generateFinishedBuilds(self, branches=[], 1813 num_builds=None, 1814 max_buildnum=None, 1815 finished_before=None, 1816 max_search=200):

1817 got = 0 1818 for Nb in itertools.count(1): 1819 if Nb > self.nextBuildNumber: 1820 break 1821 if Nb > max_search: 1822 break 1823 build = self.getBuild(-Nb) 1824 if build is None: 1825 continue 1826 if max_buildnum is not None: 1827 if build.getNumber() > max_buildnum: 1828 continue 1829 if not build.isFinished(): 1830 continue 1831 if finished_before is not None: 1832 start, end = build.getTimes() 1833 if end >= finished_before: 1834 continue 1835 if branches: 1836 if build.getSourceStamp().branch not in branches: 1837 continue 1838 got += 1 1839 yield build 1840 if num_builds is not None: 1841 if got >= num_builds: 1842 return

1843

1844 - def eventGenerator(self, branches=[], categories=[], committers=[], minTime=0):

1845 """This function creates a generator which will provide all of this 1846 Builder's status events, starting with the most recent and 1847 progressing backwards in time. """ 1848 1849 # remember the oldest-to-earliest flow here. "next" means earlier. 1850 1851 # TODO: interleave build steps and self.events by timestamp. 1852 # TODO: um, I think we're already doing that. 1853 1854 # TODO: there's probably something clever we could do here to 1855 # interleave two event streams (one from self.getBuild and the other 1856 # from self.getEvent), which would be simpler than this control flow 1857 1858 eventIndex = -1 1859 e = self.getEvent(eventIndex) 1860 for Nb in range(1, self.nextBuildNumber+1): 1861 b = self.getBuild(-Nb) 1862 if not b: 1863 # HACK: If this is the first build we are looking at, it is 1864 # possible it's in progress but locked before it has written a 1865 # pickle; in this case keep looking. 1866 if Nb == 1: 1867 continue 1868 break 1869 if b.getTimes()[0] < minTime: 1870 break 1871 if branches and not b.getSourceStamp().branch in branches: 1872 continue 1873 if categories and not b.getBuilder().getCategory() in categories: 1874 continue 1875 if committers and not [True for c in b.getChanges() if c.who in committers]: 1876 continue 1877 steps = b.getSteps() 1878 for Ns in range(1, len(steps)+1): 1879 if steps[-Ns].started: 1880 step_start = steps[-Ns].getTimes()[0] 1881 while e is not None and e.getTimes()[0] > step_start: 1882 yield e 1883 eventIndex -= 1 1884 e = self.getEvent(eventIndex) 1885 yield steps[-Ns] 1886 yield b 1887 while e is not None: 1888 yield e 1889 eventIndex -= 1 1890 e = self.getEvent(eventIndex) 1891 if e and e.getTimes()[0] < minTime: 1892 break

1893 1903

1904 - def unsubscribe(self, receiver):

1905 self.watchers.remove(receiver) 1906 self.status._builder_unsubscribe(self.name, receiver)

1907 1908 ## Builder interface (methods called by the Builder which feeds us) 1909

1910 - def setSlavenames(self, names):

1911 self.slavenames = names

1912

1913 - def addEvent(self, text=[]):

1914 # this adds a duration event. When it is done, the user should call 1915 # e.finish(). They can also mangle it by modifying .text 1916 e = Event() 1917 e.started = util.now() 1918 e.text = text 1919 self.events.append(e) 1920 return e # they are free to mangle it further

1921

1922 - def addPointEvent(self, text=[]):

1923 # this adds a point event, one which occurs as a single atomic 1924 # instant of time. 1925 e = Event() 1926 e.started = util.now() 1927 e.finished = 0 1928 e.text = text 1929 self.events.append(e) 1930 return e # for consistency, but they really shouldn't touch it

1931

1932 - def setBigState(self, state):

1933 needToUpdate = state != self.currentBigState 1934 self.currentBigState = state 1935 if needToUpdate: 1936 self.publishState()

1937

1938 - def publishState(self, target=None):

1939 state = self.currentBigState 1940 1941 if target is not None: 1942 # unicast 1943 target.builderChangedState(self.name, state) 1944 return 1945 for w in self.watchers: 1946 try: 1947 w.builderChangedState(self.name, state) 1948 except: 1949 log.msg("Exception caught publishing state to %r" % w) 1950 log.err()

1951

1952 - def newBuild(self):

1953 """The Builder has decided to start a build, but the Build object is 1954 not yet ready to report status (it has not finished creating the 1955 Steps). Create a BuildStatus object that it can use.""" 1956 number = self.nextBuildNumber 1957 self.nextBuildNumber += 1 1958 # TODO: self.saveYourself(), to make sure we don't forget about the 1959 # build number we've just allocated. This is not quite as important 1960 # as it was before we switch to determineNextBuildNumber, but I think 1961 # it may still be useful to have the new build save itself. 1962 s = BuildStatus(self, number) 1963 s.waitUntilFinished().addCallback(self._buildFinished) 1964 return s

1965 1966 # buildStarted is called by our child BuildStatus instances

1967 - def buildStarted(self, s):

1968 """Now the BuildStatus object is ready to go (it knows all of its 1969 Steps, its ETA, etc), so it is safe to notify our watchers.""" 1970 1971 assert s.builder is self # paranoia 1972 assert s.number == self.nextBuildNumber - 1 1973 assert s not in self.currentBuilds 1974 self.currentBuilds.append(s) 1975 self.touchBuildCache(s) 1976 1977 # now that the BuildStatus is prepared to answer queries, we can 1978 # announce the new build to all our watchers 1979 1980 for w in self.watchers: # TODO: maybe do this later? callLater(0)? 1981 try: 1982 receiver = w.buildStarted(self.getName(), s) 1983 if receiver: 1984 if type(receiver) == type(()): 1985 s.subscribe(receiver[0], receiver[1]) 1986 else: 1987 s.subscribe(receiver) 1988 d = s.waitUntilFinished() 1989 d.addCallback(lambda s: s.unsubscribe(receiver)) 1990 except: 1991 log.msg("Exception caught notifying %r of buildStarted event" % w) 1992 log.err()

1993

1994 - def _buildFinished(self, s):

1995 assert s in self.currentBuilds 1996 s.saveYourself() 1997 self.currentBuilds.remove(s) 1998 1999 name = self.getName() 2000 results = s.getResults() 2001 for w in self.watchers: 2002 try: 2003 w.buildFinished(name, s, results) 2004 except: 2005 log.msg("Exception caught notifying %r of buildFinished event" % w) 2006 log.err() 2007 2008 self.prune() # conserve disk

2009 2010 2011 # waterfall display (history) 2012 2013 # I want some kind of build event that holds everything about the build: 2014 # why, what changes went into it, the results of the build, itemized 2015 # test results, etc. But, I do kind of need something to be inserted in 2016 # the event log first, because intermixing step events and the larger 2017 # build event is fraught with peril. Maybe an Event-like-thing that 2018 # doesn't have a file in it but does have links. Hmm, that's exactly 2019 # what it does now. The only difference would be that this event isn't 2020 # pushed to the clients. 2021 2022 # publish to clients

2023 - def sendLastBuildStatus(self, client):

2024 #client.newLastBuildStatus(self.lastBuildStatus) 2025 pass

2026 - def sendCurrentActivityBigToEveryone(self):

2027 for s in self.subscribers: 2028 self.sendCurrentActivityBig(s)

2029 - def sendCurrentActivityBig(self, client):

2030 state = self.currentBigState 2031 if state == "offline": 2032 client.currentlyOffline() 2033 elif state == "idle": 2034 client.currentlyIdle() 2035 elif state == "building": 2036 client.currentlyBuilding() 2037 else: 2038 log.msg("Hey, self.currentBigState is weird:", state)

2039 2040 2041 ## HTML display interface 2042

2043 - def getEventNumbered(self, num):

2044 # deal with dropped events, pruned events 2045 first = self.events[0].number 2046 if first + len(self.events)-1 != self.events[-1].number: 2047 log.msg(self, 2048 "lost an event somewhere: [0] is %d, [%d] is %d" % \ 2049 (self.events[0].number, 2050 len(self.events) - 1, 2051 self.events[-1].number)) 2052 for e in self.events: 2053 log.msg("e[%d]: " % e.number, e) 2054 return None 2055 offset = num - first 2056 log.msg(self, "offset", offset) 2057 try: 2058 return self.events[offset] 2059 except IndexError: 2060 return None

2061 2062 ## Persistence of Status

2063 - def loadYourOldEvents(self):

2064 if hasattr(self, "allEvents"): 2065 # first time, nothing to get from file. Note that this is only if 2066 # the Application gets .run() . If it gets .save()'ed, then the 2067 # .allEvents attribute goes away in the initial __getstate__ and 2068 # we try to load a non-existent file. 2069 return 2070 self.allEvents = self.loadFile("events", []) 2071 if self.allEvents: 2072 self.nextEventNumber = self.allEvents[-1].number + 1 2073 else: 2074 self.nextEventNumber = 0

2075 - def saveYourOldEvents(self):

2076 self.saveFile("events", self.allEvents)

2077 2078 ## clients 2079

2080 - def addClient(self, client):

2081 if client not in self.subscribers: 2082 self.subscribers.append(client) 2083 self.sendLastBuildStatus(client) 2084 self.sendCurrentActivityBig(client) 2085 client.newEvent(self.currentSmall)

2086 - def removeClient(self, client):

2087 if client in self.subscribers: 2088 self.subscribers.remove(client)

2089

2090 - def asDict(self):

2091 result = {} 2092 # Constant 2093 # TODO(maruel): Fix me. We don't want to leak the full path. 2094 result['basedir'] = os.path.basename(self.basedir) 2095 result['category'] = self.category 2096 result['slaves'] = self.slavenames 2097 #result['url'] = self.parent.getURLForThing(self) 2098 # TODO(maruel): Add cache settings? Do we care? 2099 2100 # Transient 2101 # Collect build numbers. 2102 # Important: Only grab the *cached* builds numbers to reduce I/O. 2103 current_builds = [b.getNumber() for b in self.currentBuilds] 2104 cached_builds = list(set(self.buildCache.keys() + current_builds)) 2105 cached_builds.sort() 2106 result['cachedBuilds'] = cached_builds 2107 result['currentBuilds'] = current_builds 2108 result['state'] = self.getState()[0] 2109 # BuildRequestStatus doesn't have a number so display the SourceStamp. 2110 result['pendingBuilds'] = [ 2111 b.getSourceStamp().asDict() for b in self.getPendingBuilds() 2112 ] 2113 return result

2114 2115

2116 -class SlaveStatus:

2117 implements(interfaces.ISlaveStatus) 2118 2119 admin = None 2120 host = None 2121 access_uri = None 2122 version = None 2123 connected = False 2124 graceful_shutdown = False 2125

2126 - def __init__(self, name):

2127 self.name = name 2128 self._lastMessageReceived = 0 2129 self.runningBuilds = [] 2130 self.graceful_callbacks = []

2131

2132 - def getName(self):

2133 return self.name

2134 - def getAdmin(self):

2135 return self.admin

2136 - def getHost(self):

2137 return self.host

2138 - def getAccessURI(self):

2139 return self.access_uri

2140 - def getVersion(self):

2141 return self.version

2142 - def isConnected(self):

2143 return self.connected

2144 - def lastMessageReceived(self):

2145 return self._lastMessageReceived

2146 - def getRunningBuilds(self):

2147 return self.runningBuilds

2148

2149 - def setAdmin(self, admin):

2150 self.admin = admin

2151 - def setHost(self, host):

2152 self.host = host

2153 - def setAccessURI(self, access_uri):

2154 self.access_uri = access_uri

2155 - def setVersion(self, version):

2156 self.version = version

2157 - def setConnected(self, isConnected):

2158 self.connected = isConnected

2159 - def setLastMessageReceived(self, when):

2160 self._lastMessageReceived = when

2161

2162 - def buildStarted(self, build):

2163 self.runningBuilds.append(build)

2164 - def buildFinished(self, build):

2165 self.runningBuilds.remove(build)

2166

2167 - def getGraceful(self):

2168 """Return the graceful shutdown flag""" 2169 return self.graceful_shutdown

2170 - def setGraceful(self, graceful):

2171 """Set the graceful shutdown flag, and notify all the watchers""" 2172 self.graceful_shutdown = graceful 2173 for cb in self.graceful_callbacks: 2174 eventually(cb, graceful)

2175 - def addGracefulWatcher(self, watcher):

2176 """Add watcher to the list of watchers to be notified when the 2177 graceful shutdown flag is changed.""" 2178 if not watcher in self.graceful_callbacks: 2179 self.graceful_callbacks.append(watcher)

2180 - def removeGracefulWatcher(self, watcher):

2181 """Remove watcher from the list of watchers to be notified when the 2182 graceful shutdown flag is changed.""" 2183 if watcher in self.graceful_callbacks: 2184 self.graceful_callbacks.remove(watcher)

2185

2186 - def asDict(self):

2187 result = {} 2188 # Constant 2189 result['name'] = self.getName() 2190 result['access_uri'] = self.getAccessURI() 2191 2192 # Transient (since it changes when the slave reconnects) 2193 result['host'] = self.getHost() 2194 result['admin'] = self.getAdmin() 2195 result['version'] = self.getVersion() 2196 result['connected'] = self.isConnected() 2197 result['runningBuilds'] = [b.asDict() for b in self.getRunningBuilds()] 2198 return result

2199 2200

2201 -class Status:

2202 """ 2203 I represent the status of the buildmaster. 2204 """ 2205 implements(interfaces.IStatus) 2206

2207 - def __init__(self, botmaster, basedir):

2208 """ 2209 @type botmaster: L{buildbot.master.BotMaster} 2210 @param botmaster: the Status object uses C{.botmaster} to get at 2211 both the L{buildbot.master.BuildMaster} (for 2212 various buildbot-wide parameters) and the 2213 actual Builders (to get at their L{BuilderStatus} 2214 objects). It is not allowed to change or influence 2215 anything through this reference. 2216 @type basedir: string 2217 @param basedir: this provides a base directory in which saved status 2218 information (changes.pck, saved Build status 2219 pickles) can be stored 2220 """ 2221 self.botmaster = botmaster 2222 self.db = None 2223 self.basedir = basedir 2224 self.watchers = [] 2225 assert os.path.isdir(basedir) 2226 # compress logs bigger than 4k, a good default on linux 2227 self.logCompressionLimit = 4*1024 2228 self.logCompressionMethod = "bz2" 2229 # No default limit to the log size 2230 self.logMaxSize = None 2231 self.logMaxTailSize = None 2232 2233 self._builder_observers = collections.KeyedSets() 2234 self._buildreq_observers = collections.KeyedSets() 2235 self._buildset_success_waiters = collections.KeyedSets() 2236 self._buildset_finished_waiters = collections.KeyedSets()

2237

2238 - def setDB(self, db):

2239 self.db = db 2240 self.db.subscribe_to("add-build", self._db_builds_changed) 2241 self.db.subscribe_to("add-buildset", self._db_buildset_added) 2242 self.db.subscribe_to("modify-buildset", self._db_buildsets_changed) 2243 self.db.subscribe_to("add-buildrequest", self._db_buildrequest_added) 2244 self.db.subscribe_to("cancel-buildrequest", self._db_buildrequest_cancelled)

2245 2246 # methods called by our clients 2247

2248 - def getProjectName(self):

2249 return self.botmaster.parent.projectName

2250 - def getProjectURL(self):

2251 return self.botmaster.parent.projectURL

2252 - def getBuildbotURL(self):

2253 return self.botmaster.parent.buildbotURL

2254

2255 - def getURLForThing(self, thing):

2256 prefix = self.getBuildbotURL() 2257 if not prefix: 2258 return None 2259 if interfaces.IStatus.providedBy(thing): 2260 return prefix 2261 if interfaces.ISchedulerStatus.providedBy(thing): 2262 pass 2263 if interfaces.IBuilderStatus.providedBy(thing): 2264 builder = thing 2265 return prefix + "builders/%s" % ( 2266 urllib.quote(builder.getName(), safe=''), 2267 ) 2268 if interfaces.IBuildStatus.providedBy(thing): 2269 build = thing 2270 builder = build.getBuilder() 2271 return prefix + "builders/%s/builds/%d" % ( 2272 urllib.quote(builder.getName(), safe=''), 2273 build.getNumber()) 2274 if interfaces.IBuildStepStatus.providedBy(thing): 2275 step = thing 2276 build = step.getBuild() 2277 builder = build.getBuilder() 2278 return prefix + "builders/%s/builds/%d/steps/%s" % ( 2279 urllib.quote(builder.getName(), safe=''), 2280 build.getNumber(), 2281 urllib.quote(step.getName(), safe='')) 2282 # IBuildSetStatus 2283 # IBuildRequestStatus 2284 # ISlaveStatus 2285 2286 # IStatusEvent 2287 if interfaces.IStatusEvent.providedBy(thing): 2288 from buildbot.changes import changes 2289 # TODO: this is goofy, create IChange or something 2290 if isinstance(thing, changes.Change): 2291 change = thing 2292 return "%schanges/%d" % (prefix, change.number) 2293 2294 if interfaces.IStatusLog.providedBy(thing): 2295 log = thing 2296 step = log.getStep() 2297 build = step.getBuild() 2298 builder = build.getBuilder() 2299 2300 logs = step.getLogs() 2301 for i in range(len(logs)): 2302 if log is logs[i]: 2303 break 2304 else: 2305 return None 2306 return prefix + "builders/%s/builds/%d/steps/%s/logs/%s" % ( 2307 urllib.quote(builder.getName(), safe=''), 2308 build.getNumber(), 2309 urllib.quote(step.getName(), safe=''), 2310 urllib.quote(log.getName()))

2311

2312 - def getChangeSources(self):

2313 return list(self.botmaster.parent.change_svc)

2314

2315 - def getChange(self, number):

2316 return self.botmaster.parent.change_svc.getChangeNumberedNow(number)

2317

2318 - def getSchedulers(self):

2319 return self.botmaster.parent.allSchedulers()

2320

2321 - def getBuilderNames(self, categories=None):

2322 if categories == None: 2323 return self.botmaster.builderNames[:] # don't let them break it 2324 2325 l = [] 2326 # respect addition order 2327 for name in self.botmaster.builderNames: 2328 builder = self.botmaster.builders[name] 2329 if builder.builder_status.category in categories: 2330 l.append(name) 2331 return l

2332

2333 - def getBuilder(self, name):

2334 """ 2335 @rtype: L{BuilderStatus} 2336 """ 2337 return self.botmaster.builders[name].builder_status

2338

2339 - def getSlaveNames(self):

2340 return self.botmaster.slaves.keys()

2341

2342 - def getSlave(self, slavename):

2343 return self.botmaster.slaves[slavename].slave_status

2344

2345 - def getBuildSets(self):

2346 return [BuildSetStatus(bsid, self, self.db) 2347 for bsid in self.db.get_active_buildset_ids()]

2348

2349 - def generateFinishedBuilds(self, builders=[], branches=[], 2350 num_builds=None, finished_before=None, 2351 max_search=200):

2352 2353 def want_builder(bn): 2354 if builders: 2355 return bn in builders 2356 return True

2357 builder_names = [bn 2358 for bn in self.getBuilderNames() 2359 if want_builder(bn)] 2360 2361 # 'sources' is a list of generators, one for each Builder we're 2362 # using. When the generator is exhausted, it is replaced in this list 2363 # with None. 2364 sources = [] 2365 for bn in builder_names: 2366 b = self.getBuilder(bn) 2367 g = b.generateFinishedBuilds(branches, 2368 finished_before=finished_before, 2369 max_search=max_search) 2370 sources.append(g) 2371 2372 # next_build the next build from each source 2373 next_build = [None] * len(sources) 2374 2375 def refill(): 2376 for i,g in enumerate(sources): 2377 if next_build[i]: 2378 # already filled 2379 continue 2380 if not g: 2381 # already exhausted 2382 continue 2383 try: 2384 next_build[i] = g.next() 2385 except StopIteration: 2386 next_build[i] = None 2387 sources[i] = None

2388 2389 got = 0 2390 while True: 2391 refill() 2392 # find the latest build among all the candidates 2393 candidates = [(i, b, b.getTimes()[1]) 2394 for i,b in enumerate(next_build) 2395 if b is not None] 2396 candidates.sort(lambda x,y: cmp(x[2], y[2])) 2397 if not candidates: 2398 return 2399 2400 # and remove it from the list 2401 i, build, finshed_time = candidates[-1] 2402 next_build[i] = None 2403 got += 1 2404 yield build 2405 if num_builds is not None: 2406 if got >= num_builds: 2407 return 2408

2413 - def unsubscribe(self, target):

2414 self.watchers.remove(target)

2415 2416 2417 # methods called by upstream objects 2418

2419 - def announceNewBuilder(self, target, name, builder_status):

2420 t = target.builderAdded(name, builder_status) 2421 if t: 2422 builder_status.subscribe(t)

2423

2424 - def builderAdded(self, name, basedir, category=None):

2425 """ 2426 @rtype: L{BuilderStatus} 2427 """ 2428 filename = os.path.join(self.basedir, basedir, "builder") 2429 log.msg("trying to load status pickle from %s" % filename) 2430 builder_status = None 2431 try: 2432 builder_status = load(open(filename, "rb")) 2433 styles.doUpgrade() 2434 except IOError: 2435 log.msg("no saved status pickle, creating a new one") 2436 except: 2437 log.msg("error while loading status pickle, creating a new one") 2438 log.msg("error follows:") 2439 log.err() 2440 if not builder_status: 2441 builder_status = BuilderStatus(name, category) 2442 builder_status.addPointEvent(["builder", "created"]) 2443 log.msg("added builder %s in category %s" % (name, category)) 2444 # an unpickled object might not have category set from before, 2445 # so set it here to make sure 2446 builder_status.category = category 2447 builder_status.basedir = os.path.join(self.basedir, basedir) 2448 builder_status.name = name # it might have been updated 2449 builder_status.status = self 2450 2451 if not os.path.isdir(builder_status.basedir): 2452 os.makedirs(builder_status.basedir) 2453 builder_status.determineNextBuildNumber() 2454 2455 builder_status.setBigState("offline") 2456 builder_status.setLogCompressionLimit(self.logCompressionLimit) 2457 builder_status.setLogCompressionMethod(self.logCompressionMethod) 2458 builder_status.setLogMaxSize(self.logMaxSize) 2459 builder_status.setLogMaxTailSize(self.logMaxTailSize) 2460 2461 for t in self.watchers: 2462 self.announceNewBuilder(t, name, builder_status) 2463 2464 return builder_status

2465

2466 - def builderRemoved(self, name):

2467 for t in self.watchers: 2468 if hasattr(t, 'builderRemoved'): 2469 t.builderRemoved(name)

2470

2471 - def slaveConnected(self, name):

2472 for t in self.watchers: 2473 if hasattr(t, 'slaveConnected'): 2474 t.slaveConnected(name)

2475

2476 - def slaveDisconnected(self, name):

2477 for t in self.watchers: 2478 if hasattr(t, 'slaveDisconnected'): 2479 t.slaveDisconnected(name)

2480

2481 - def changeAdded(self, change):

2482 for t in self.watchers: 2483 if hasattr(t, 'changeAdded'): 2484 t.changeAdded(change)

2485

2486 - def asDict(self):

2487 result = {} 2488 # Constant 2489 result['projectName'] = self.getProjectName() 2490 result['projectURL'] = self.getProjectURL() 2491 result['buildbotURL'] = self.getBuildbotURL() 2492 # TODO: self.getSchedulers() 2493 # self.getChangeSources() 2494 return result

2495

2496 - def buildreqs_retired(self, requests):

2497 for r in requests: 2498 #r.id: notify subscribers (none right now) 2499 # r.bsid: check for completion, notify subscribers, unsubscribe 2500 pass

2501

2502 - def get_buildreq_for_id(self, brid):

2503 return BuildRequestStatus(brid, self, self.db)

2504

2505 - def _db_builds_changed(self, category, bid):

2506 brid,buildername,buildnum = self.db.get_build_info(bid) 2507 if brid in self._buildreq_observers: 2508 bs = self.getBuilder(buildername).getBuild(buildnum) 2509 if bs: 2510 for o in self._buildreq_observers[brid]: 2511 eventually(o, bs)

2512

2513 - def _buildrequest_subscribe(self, brid, observer):

2514 self._buildreq_observers.add(brid, observer)

2515

2516 - def _buildrequest_unsubscribe(self, brid, observer):

2517 self._buildreq_observers.discard(brid, observer)

2518

2519 - def _db_buildset_added(self, category, bsid):

2520 bss = BuildSetStatus(bsid, self, self.db) 2521 for t in self.watchers: 2522 if hasattr(t, 'buildsetSubmitted'): 2523 t.buildsetSubmitted(bss)

2524

2525 - def _buildset_waitUntilSuccess(self, bsid):

2526 d = defer.Deferred() 2527 self._buildset_success_waiters.add(bsid, d) 2528 # now check for a buildset which was already successful 2529 self._db_buildsets_changed("modify-buildset", bsid) 2530 return d

2531 - def _buildset_waitUntilFinished(self, bsid):

2532 d = defer.Deferred() 2533 self._buildset_finished_waiters.add(bsid, d) 2534 self._db_buildsets_changed("modify-buildset", bsid) 2535 return d

2536

2537 - def _db_buildsets_changed(self, category, *bsids):

2538 for bsid in bsids: 2539 self._db_buildset_changed(bsid)

2540

2541 - def _db_buildset_changed(self, bsid):

2542 # check bsid to see if it's successful or finished, and notify anyone 2543 # who cares 2544 if (bsid not in self._buildset_success_waiters 2545 and bsid not in self._buildset_finished_waiters): 2546 return 2547 successful,finished = self.db.examine_buildset(bsid) 2548 bss = BuildSetStatus(bsid, self, self.db) 2549 if successful is not None: 2550 for d in self._buildset_success_waiters.pop(bsid): 2551 eventually(d.callback, bss) 2552 if finished: 2553 for d in self._buildset_finished_waiters.pop(bsid): 2554 eventually(d.callback, bss)

2555

2556 - def _builder_subscribe(self, buildername, watcher):

2557 # should get requestSubmitted and requestCancelled 2558 self._builder_observers.add(buildername, watcher)

2559

2560 - def _builder_unsubscribe(self, buildername, watcher):

2561 self._builder_observers.discard(buildername, watcher)

2562

2563 - def _db_buildrequest_added(self, category, *brids):

2564 self._handle_buildrequest_event("added", brids)

2565 - def _db_buildrequest_cancelled(self, category, *brids):

2566 self._handle_buildrequest_event("cancelled", brids)

2567 - def _handle_buildrequest_event(self, mode, brids):

2568 for brid in brids: 2569 buildername = self.db.get_buildername_for_brid(brid) 2570 if buildername in self._builder_observers: 2571 brs = BuildRequestStatus(brid, self, self.db) 2572 for observer in self._builder_observers[buildername]: 2573 if mode == "added": 2574 if hasattr(observer, 'requestSubmitted'): 2575 eventually(observer.requestSubmitted, brs) 2576 else: 2577 if hasattr(observer, 'requestCancelled'): 2578 eventually(observer.requestCancelled, brs)

2579 2580 2581 2582 # vim: set ts=4 sts=4 sw=4 et: 2583

Source Code for Module buildbot.status.builder