1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 import weakref
18 import os, re, itertools
19 from cPickle import load, dump
20
21 from zope.interface import implements
22 from twisted.python import log, runtime
23 from twisted.persisted import styles
24 from buildbot.process import metrics
25 from buildbot import interfaces, util
26 from buildbot.status.event import Event
27 from buildbot.status.build import BuildStatus
28 from buildbot.status.buildrequest import BuildRequestStatus
29
30
31 from buildbot.status.results import SUCCESS, WARNINGS, FAILURE, SKIPPED
32 from buildbot.status.results import EXCEPTION, RETRY, Results, worst_status
33 _hush_pyflakes = [ SUCCESS, WARNINGS, FAILURE, SKIPPED,
34 EXCEPTION, RETRY, Results, worst_status ]
35
37 """I handle status information for a single process.build.Builder object.
38 That object sends status changes to me (frequently as Events), and I
39 provide them on demand to the various status recipients, like the HTML
40 waterfall display and the live status clients. It also sends build
41 summaries to me, which I log and provide to status clients who aren't
42 interested in seeing details of the individual build steps.
43
44 I am responsible for maintaining the list of historic Events and Builds,
45 pruning old ones, and loading them from / saving them to disk.
46
47 I live in the buildbot.process.build.Builder object, in the
48 .builder_status attribute.
49
50 @type category: string
51 @ivar category: user-defined category this builder belongs to; can be
52 used to filter on in status clients
53 """
54
55 implements(interfaces.IBuilderStatus, interfaces.IEventSource)
56
57 persistenceVersion = 1
58 persistenceForgets = ( 'wasUpgraded', )
59
60
61
62
63 buildCacheSize = 15
64 eventHorizon = 50
65
66
67 logHorizon = 40
68 buildHorizon = 100
69
70 category = None
71 currentBigState = "offline"
72 basedir = None
73
74 - def __init__(self, buildername, category=None):
75 self.name = buildername
76 self.category = category
77
78 self.slavenames = []
79 self.events = []
80
81
82 self.lastBuildStatus = None
83
84
85 self.currentBuilds = []
86 self.nextBuild = None
87 self.watchers = []
88 self.buildCache = weakref.WeakValueDictionary()
89 self.buildCache_LRU = []
90 self.logCompressionLimit = False
91 self.logCompressionMethod = "bz2"
92 self.logMaxSize = None
93 self.logMaxTailSize = None
94
95
96
98
99
100
101
102 d = styles.Versioned.__getstate__(self)
103 d['watchers'] = []
104 del d['buildCache']
105 del d['buildCache_LRU']
106 for b in self.currentBuilds:
107 b.saveYourself()
108
109 del d['currentBuilds']
110 d.pop('pendingBuilds', None)
111 del d['currentBigState']
112 del d['basedir']
113 del d['status']
114 del d['nextBuildNumber']
115 return d
116
118
119
120 styles.Versioned.__setstate__(self, d)
121 self.buildCache = weakref.WeakValueDictionary()
122 self.buildCache_LRU = []
123 self.currentBuilds = []
124 self.watchers = []
125 self.slavenames = []
126
127
128
134
136 if hasattr(self, 'slavename'):
137 self.slavenames = [self.slavename]
138 del self.slavename
139 if hasattr(self, 'nextBuildNumber'):
140 del self.nextBuildNumber
141 self.wasUpgraded = True
142
144 """Scan our directory of saved BuildStatus instances to determine
145 what our self.nextBuildNumber should be. Set it one larger than the
146 highest-numbered build we discover. This is called by the top-level
147 Status object shortly after we are created or loaded from disk.
148 """
149 existing_builds = [int(f)
150 for f in os.listdir(self.basedir)
151 if re.match("^\d+$", f)]
152 if existing_builds:
153 self.nextBuildNumber = max(existing_builds) + 1
154 else:
155 self.nextBuildNumber = 0
156
158 self.logCompressionLimit = lowerLimit
159
161 assert method in ("bz2", "gz")
162 self.logCompressionMethod = method
163
166
169
188
189
190
191
194
201
242
243 - def prune(self, events_only=False):
244
245 self.events = self.events[-self.eventHorizon:]
246
247 if events_only:
248 return
249
250
251 if self.buildHorizon is not None:
252 earliest_build = self.nextBuildNumber - self.buildHorizon
253 else:
254 earliest_build = 0
255
256 if self.logHorizon is not None:
257 earliest_log = self.nextBuildNumber - self.logHorizon
258 else:
259 earliest_log = 0
260
261 if earliest_log < earliest_build:
262 earliest_log = earliest_build
263
264 if earliest_build == 0:
265 return
266
267
268 build_re = re.compile(r"^([0-9]+)$")
269 build_log_re = re.compile(r"^([0-9]+)-.*$")
270
271 if not os.path.exists(self.basedir):
272 return
273
274 for filename in os.listdir(self.basedir):
275 num = None
276 mo = build_re.match(filename)
277 is_logfile = False
278 if mo:
279 num = int(mo.group(1))
280 else:
281 mo = build_log_re.match(filename)
282 if mo:
283 num = int(mo.group(1))
284 is_logfile = True
285
286 if num is None: continue
287 if num in self.buildCache: continue
288
289 if (is_logfile and num < earliest_log) or num < earliest_build:
290 pathname = os.path.join(self.basedir, filename)
291 log.msg("pruning '%s'" % pathname)
292 try: os.unlink(pathname)
293 except OSError: pass
294
295
298
301
304
313 d.addCallback(make_statuses)
314 return d
315
317 return self.currentBuilds
318
324
327
338
340 try:
341 return self.events[number]
342 except IndexError:
343 return None
344
345 - def generateFinishedBuilds(self, branches=[],
346 num_builds=None,
347 max_buildnum=None,
348 finished_before=None,
349 max_search=200):
350 got = 0
351 for Nb in itertools.count(1):
352 if Nb > self.nextBuildNumber:
353 break
354 if Nb > max_search:
355 break
356 build = self.getBuild(-Nb)
357 if build is None:
358 continue
359 if max_buildnum is not None:
360 if build.getNumber() > max_buildnum:
361 continue
362 if not build.isFinished():
363 continue
364 if finished_before is not None:
365 start, end = build.getTimes()
366 if end >= finished_before:
367 continue
368 if branches:
369 if build.getSourceStamp().branch not in branches:
370 continue
371 got += 1
372 yield build
373 if num_builds is not None:
374 if got >= num_builds:
375 return
376
377 - def eventGenerator(self, branches=[], categories=[], committers=[], minTime=0):
378 """This function creates a generator which will provide all of this
379 Builder's status events, starting with the most recent and
380 progressing backwards in time. """
381
382
383
384
385
386
387
388
389
390
391 eventIndex = -1
392 e = self.getEvent(eventIndex)
393 for Nb in range(1, self.nextBuildNumber+1):
394 b = self.getBuild(-Nb)
395 if not b:
396
397
398
399 if Nb == 1:
400 continue
401 break
402 if b.getTimes()[0] < minTime:
403 break
404 if branches and not b.getSourceStamp().branch in branches:
405 continue
406 if categories and not b.getBuilder().getCategory() in categories:
407 continue
408 if committers and not [True for c in b.getChanges() if c.who in committers]:
409 continue
410 steps = b.getSteps()
411 for Ns in range(1, len(steps)+1):
412 if steps[-Ns].started:
413 step_start = steps[-Ns].getTimes()[0]
414 while e is not None and e.getTimes()[0] > step_start:
415 yield e
416 eventIndex -= 1
417 e = self.getEvent(eventIndex)
418 yield steps[-Ns]
419 yield b
420 while e is not None:
421 yield e
422 eventIndex -= 1
423 e = self.getEvent(eventIndex)
424 if e and e.getTimes()[0] < minTime:
425 break
426
436
440
441
442
444 self.slavenames = names
445
455
466
472
486
488 """The Builder has decided to start a build, but the Build object is
489 not yet ready to report status (it has not finished creating the
490 Steps). Create a BuildStatus object that it can use."""
491 number = self.nextBuildNumber
492 self.nextBuildNumber += 1
493
494
495
496
497 s = BuildStatus(self, number)
498 s.waitUntilFinished().addCallback(self._buildFinished)
499 return s
500
501
503 """Now the BuildStatus object is ready to go (it knows all of its
504 Steps, its ETA, etc), so it is safe to notify our watchers."""
505
506 assert s.builder is self
507 assert s not in self.currentBuilds
508 self.currentBuilds.append(s)
509 self.touchBuildCache(s)
510
511
512
513
514 for w in self.watchers:
515 try:
516 receiver = w.buildStarted(self.getName(), s)
517 if receiver:
518 if type(receiver) == type(()):
519 s.subscribe(receiver[0], receiver[1])
520 else:
521 s.subscribe(receiver)
522 d = s.waitUntilFinished()
523 d.addCallback(lambda s: s.unsubscribe(receiver))
524 except:
525 log.msg("Exception caught notifying %r of buildStarted event" % w)
526 log.err()
527
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
560
561 first = self.events[0].number
562 if first + len(self.events)-1 != self.events[-1].number:
563 log.msg(self,
564 "lost an event somewhere: [0] is %d, [%d] is %d" % \
565 (self.events[0].number,
566 len(self.events) - 1,
567 self.events[-1].number))
568 for e in self.events:
569 log.msg("e[%d]: " % e.number, e)
570 return None
571 offset = num - first
572 log.msg(self, "offset", offset)
573 try:
574 return self.events[offset]
575 except IndexError:
576 return None
577
578
580 if hasattr(self, "allEvents"):
581
582
583
584
585 return
586 self.allEvents = self.loadFile("events", [])
587 if self.allEvents:
588 self.nextEventNumber = self.allEvents[-1].number + 1
589 else:
590 self.nextEventNumber = 0
592 self.saveFile("events", self.allEvents)
593
594
595
597 if client not in self.subscribers:
598 self.subscribers.append(client)
599 self.sendCurrentActivityBig(client)
600 client.newEvent(self.currentSmall)
604
606 result = {}
607
608
609 result['basedir'] = os.path.basename(self.basedir)
610 result['category'] = self.category
611 result['slaves'] = self.slavenames
612
613
614
615
616
617
618 current_builds = [b.getNumber() for b in self.currentBuilds]
619 cached_builds = list(set(self.buildCache.keys() + current_builds))
620 cached_builds.sort()
621 result['cachedBuilds'] = cached_builds
622 result['currentBuilds'] = current_builds
623 result['state'] = self.getState()[0]
624
625
626 result['pendingBuilds'] = 0
627 return result
628
630 """Just like L{asDict}, but with a nonzero pendingBuilds."""
631 result = self.asDict()
632 d = self.getPendingBuildRequestStatuses()
633 def combine(statuses):
634 result['pendingBuilds'] = len(statuses)
635 return result
636 d.addCallback(combine)
637 return d
638
641
642
643