1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 import time
17 import operator
18 import re
19 import urllib
20 from twisted.internet import defer
21 from buildbot import util
22 from buildbot.status import builder
23 from buildbot.status.web.base import HtmlResource
24 from buildbot.changes import changes
27
29 """Given the current and past results, return the class that will be used
30 by the css to display the right color for a box."""
31
32 if inProgress:
33 return "running"
34
35 if results is None:
36 return "notstarted"
37
38 if results == builder.SUCCESS:
39 return "success"
40
41 if results == builder.WARNINGS:
42 return "warnings"
43
44 if results == builder.FAILURE:
45 if not prevResults:
46
47
48 return "failure"
49
50 if prevResults != builder.FAILURE:
51
52 return "failure"
53 else:
54
55 return "warnings"
56
57
58 return "exception"
59
61
63 """Helper class that contains all the information we need for a revision."""
64
74
77 """Helper class that contains all the information we need for a build."""
78
79 - def __init__(self, revision, build, details):
89
92 """Main console class. It displays a user-oriented status page.
93 Every change is a line in the page, and it shows the result of the first
94 build with this change for each slave."""
95
105
106 - def getPageTitle(self, request):
107 status = self.getStatus(request)
108 title = status.getTitle()
109 if title:
110 return "BuildBot: %s" % title
111 else:
112 return "BuildBot"
113
116
117
118
119
120
132
133 - def fetchChangesFromHistory(self, status, max_depth, max_builds, debugInfo):
134 """Look at the history of the builders and try to fetch as many changes
135 as possible. We need this when the main source does not contain enough
136 sourcestamps.
137
138 max_depth defines how many builds we will parse for a given builder.
139 max_builds defines how many builds total we want to parse. This is to
140 limit the amount of time we spend in this function.
141
142 This function is sub-optimal, but the information returned by this
143 function is cached, so this function won't be called more than once.
144 """
145
146 allChanges = list()
147 build_count = 0
148 for builderName in status.getBuilderNames()[:]:
149 if build_count > max_builds:
150 break
151
152 builder = status.getBuilder(builderName)
153 build = self.getHeadBuild(builder)
154 depth = 0
155 while build and depth < max_depth and build_count < max_builds:
156 depth += 1
157 build_count += 1
158 sourcestamp = build.getSourceStamp()
159 allChanges.extend(sourcestamp.changes[:])
160 build = build.getPreviousBuild()
161
162 debugInfo["source_fetch_len"] = len(allChanges)
163 return allChanges
164
165 @defer.deferredGenerator
195
197 """Returns an HTML list of failures for a given build."""
198 details = {}
199 if not build.getLogs():
200 return details
201
202 for step in build.getSteps():
203 (result, reason) = step.getResults()
204 if result == builder.FAILURE:
205 name = step.getName()
206
207
208 stripHtml = re.compile(r'<.*?>')
209 strippedDetails = stripHtml.sub('', ' '.join(step.getText()))
210
211 details['buildername'] = builderName
212 details['status'] = strippedDetails
213 details['reason'] = reason
214 logs = details['logs'] = []
215
216 if step.getLogs():
217 for log in step.getLogs():
218 logname = log.getName()
219 logurl = request.childLink(
220 "../builders/%s/builds/%s/steps/%s/logs/%s" %
221 (urllib.quote(builderName),
222 build.getNumber(),
223 urllib.quote(name),
224 urllib.quote(logname)))
225 logs.append(dict(url=logurl, name=logname))
226 return details
227
228 - def getBuildsForRevision(self, request, builder, builderName, lastRevision,
229 numBuilds, debugInfo):
230 """Return the list of all the builds for a given builder that we will
231 need to be able to display the console page. We start by the most recent
232 build, and we go down until we find a build that was built prior to the
233 last change we are interested in."""
234
235 revision = lastRevision
236
237 builds = []
238 build = self.getHeadBuild(builder)
239 number = 0
240 while build and number < numBuilds:
241 debugInfo["builds_scanned"] += 1
242 number += 1
243
244
245
246
247 got_rev = -1
248 try:
249 got_rev = build.getProperty("got_revision")
250 if not self.comparator.isValidRevision(got_rev):
251 got_rev = -1
252 except KeyError:
253 pass
254
255 try:
256 if got_rev == -1:
257 got_rev = build.getProperty("revision")
258 if not self.comparator.isValidRevision(got_rev):
259 got_rev = -1
260 except:
261 pass
262
263
264
265
266
267 if got_rev and got_rev != -1:
268 details = self.getBuildDetails(request, builderName, build)
269 devBuild = DevBuild(got_rev, build, details)
270 builds.append(devBuild)
271
272
273 current_revision = self.getChangeForBuild(
274 build, revision)
275 if self.comparator.isRevisionEarlier(
276 devBuild, current_revision):
277 break
278
279 build = build.getPreviousBuild()
280
281 return builds
282
295
298 """Returns a dictionary of builds we need to inspect to be able to
299 display the console page. The key is the builder name, and the value is
300 an array of build we care about. We also returns a dictionary of
301 builders we care about. The key is it's category.
302
303 lastRevision is the last revision we want to display in the page.
304 categories is a list of categories to display. It is coming from the
305 HTTP GET parameters.
306 builders is a list of builders to display. It is coming from the HTTP
307 GET parameters.
308 """
309
310 allBuilds = dict()
311
312
313 builderList = dict()
314
315 debugInfo["builds_scanned"] = 0
316
317 builderNames = status.getBuilderNames()[:]
318 for builderName in builderNames:
319 builder = status.getBuilder(builderName)
320
321
322 if categories and builder.category not in categories:
323 continue
324 if builders and builderName not in builders:
325 continue
326
327
328 category = builder.category or "default"
329
330
331
332
333
334 category = category.split('|')[0]
335 if not builderList.get(category):
336 builderList[category] = []
337
338
339 builderList[category].append(builderName)
340
341 allBuilds[builderName] = self.getBuildsForRevision(request,
342 builder,
343 builderName,
344 lastRevision,
345 numBuilds,
346 debugInfo)
347
348 return (builderList, allBuilds)
349
350
351
352
353
354
356 """Display the top category line."""
357
358 count = 0
359 for category in builderList:
360 count += len(builderList[category])
361
362 categories = builderList.keys()
363 categories.sort()
364
365 cs = []
366
367 for category in categories:
368 c = {}
369
370 c["name"] = category
371
372
373
374
375 c["size"] = (len(builderList[category]) * 100) / count
376 cs.append(c)
377
378 return cs
379
424
426 """Display the boxes that represent the status of each builder in the
427 first build "revision" was in. Returns an HTML list of errors that
428 happened during these builds."""
429
430 details = []
431 nbSlaves = 0
432 for category in builderList:
433 nbSlaves += len(builderList[category])
434
435
436 categories = builderList.keys()
437 categories.sort()
438
439 builds = {}
440
441
442 for category in categories:
443
444 builds[category] = []
445
446
447 for builder in builderList[category]:
448 introducedIn = None
449 firstNotIn = None
450
451
452 for build in allBuilds[builder]:
453 if self.comparator.isRevisionEarlier(build, revision):
454 firstNotIn = build
455 break
456 else:
457 introducedIn = build
458
459
460
461 results = None
462 previousResults = None
463 if introducedIn:
464 results = introducedIn.results
465 if firstNotIn:
466 previousResults = firstNotIn.results
467
468 isRunning = False
469 if introducedIn and not introducedIn.isFinished:
470 isRunning = True
471
472 url = "./waterfall"
473 pageTitle = builder
474 tag = ""
475 current_details = {}
476 if introducedIn:
477 current_details = introducedIn.details or ""
478 url = "./buildstatus?builder=%s&number=%s" % (urllib.quote(builder),
479 introducedIn.number)
480 pageTitle += " "
481 pageTitle += urllib.quote(' '.join(introducedIn.text), ' \n\\/:')
482
483 builderStrip = builder.replace(' ', '')
484 builderStrip = builderStrip.replace('(', '')
485 builderStrip = builderStrip.replace(')', '')
486 builderStrip = builderStrip.replace('.', '')
487 tag = "Tag%s%s" % (builderStrip, introducedIn.number)
488
489 if isRunning:
490 pageTitle += ' ETA: %ds' % (introducedIn.eta or 0)
491
492 resultsClass = getResultsClass(results, previousResults, isRunning)
493
494 b = {}
495 b["url"] = url
496 b["pageTitle"] = pageTitle
497 b["color"] = resultsClass
498 b["tag"] = tag
499
500 builds[category].append(b)
501
502
503
504 if current_details and resultsClass == "failure":
505 details.append(current_details)
506
507 return (builds, details)
508
510 """Filter a set of revisions based on any number of filter criteria.
511 If specified, filter should be a dict with keys corresponding to
512 revision attributes, and values of 1+ strings"""
513 if not filter:
514 if max_revs is None:
515 for rev in reversed(revisions):
516 yield DevRevision(rev)
517 else:
518 for index,rev in enumerate(reversed(revisions)):
519 if index >= max_revs:
520 break
521 yield DevRevision(rev)
522 else:
523 for index, rev in enumerate(reversed(revisions)):
524 if max_revs and index >= max_revs:
525 break
526 try:
527 for field,acceptable in filter.iteritems():
528 if not hasattr(rev, field):
529 raise DoesNotPassFilter
530 if type(acceptable) in (str, unicode):
531 if getattr(rev, field) != acceptable:
532 raise DoesNotPassFilter
533 elif type(acceptable) in (list, tuple, set):
534 if getattr(rev, field) not in acceptable:
535 raise DoesNotPassFilter
536 yield DevRevision(rev)
537 except DoesNotPassFilter:
538 pass
539
540 - def displayPage(self, request, status, builderList, allBuilds, revisions,
541 categories, repository, branch, debugInfo):
542 """Display the console page."""
543
544 subs = dict()
545 subs["branch"] = branch or 'trunk'
546 subs["repository"] = repository
547 if categories:
548 subs["categories"] = ' '.join(categories)
549 subs["time"] = time.strftime("%a %d %b %Y %H:%M:%S",
550 time.localtime(util.now()))
551 subs["debugInfo"] = debugInfo
552 subs["ANYBRANCH"] = ANYBRANCH
553
554 if builderList:
555 subs["categories"] = self.displayCategories(builderList, debugInfo)
556 subs['slaves'] = self.displaySlaveLine(status, builderList, debugInfo)
557 else:
558 subs["categories"] = []
559
560 subs['revisions'] = []
561
562
563 for revision in revisions:
564 r = {}
565
566
567 r['id'] = revision.revision
568 r['link'] = revision.revlink
569 r['who'] = revision.who
570 r['date'] = revision.date
571 r['comments'] = revision.comments
572 r['repository'] = revision.repository
573 r['project'] = revision.project
574
575
576 (builds, details) = self.displayStatusLine(builderList,
577 allBuilds,
578 revision,
579 debugInfo)
580 r['builds'] = builds
581 r['details'] = details
582
583
584 r["span"] = len(builderList) + 2
585
586 subs['revisions'].append(r)
587
588
589
590
591 debugInfo["load_time"] = time.time() - debugInfo["load_time"]
592 return subs
593
594
595 - def content(self, request, cxt):
596 "This method builds the main console view display."
597
598 reload_time = None
599
600
601 if "reload" in request.args:
602 try:
603 reload_time = int(request.args["reload"][0])
604 if reload_time != 0:
605 reload_time = max(reload_time, 15)
606 except ValueError:
607 pass
608
609 request.setHeader('Cache-Control', 'no-cache')
610
611
612 if not reload_time:
613 reload_time = 60
614
615
616 if reload_time is not None and reload_time != 0:
617 cxt['refresh'] = reload_time
618
619
620 debugInfo = cxt['debuginfo'] = dict()
621 debugInfo["load_time"] = time.time()
622
623
624
625 categories = request.args.get("category", [])
626
627 builders = request.args.get("builder", [])
628
629 repository = request.args.get("repository", [None])[0]
630
631 branch = request.args.get("branch", [ANYBRANCH])[0]
632
633 devName = request.args.get("name", [])
634
635
636 status = self.getStatus(request)
637
638
639
640
641
642 numRevs = int(request.args.get("revs", [40])[0])
643 if devName:
644 numRevs *= 2
645 numBuilds = numRevs
646
647
648
649 d = self.getAllChanges(request, status, debugInfo)
650 def got_changes(allChanges):
651 debugInfo["source_all"] = len(allChanges)
652
653 revFilter = {}
654 if branch != ANYBRANCH:
655 revFilter['branch'] = branch
656 if devName:
657 revFilter['who'] = devName
658 if repository:
659 revFilter['repository'] = repository
660 revisions = list(self.filterRevisions(allChanges, max_revs=numRevs,
661 filter=revFilter))
662 debugInfo["revision_final"] = len(revisions)
663
664
665
666 builderList = None
667 allBuilds = None
668 if revisions:
669 lastRevision = revisions[len(revisions) - 1].revision
670 debugInfo["last_revision"] = lastRevision
671
672 (builderList, allBuilds) = self.getAllBuildsForRevision(status,
673 request,
674 lastRevision,
675 numBuilds,
676 categories,
677 builders,
678 debugInfo)
679
680 debugInfo["added_blocks"] = 0
681
682 cxt.update(self.displayPage(request, status, builderList,
683 allBuilds, revisions, categories,
684 repository, branch, debugInfo))
685
686 templates = request.site.buildbot_service.templates
687 template = templates.get_template("console.html")
688 data = template.render(cxt)
689 return data
690 d.addCallback(got_changes)
691 return d
692
694 """Used for comparing between revisions, as some
695 VCS use a plain counter for revisions (like SVN)
696 while others use different concepts (see Git).
697 """
698
699
700
702 """Used for comparing 2 changes"""
703 raise NotImplementedError
704
706 """Checks whether the revision seems like a VCS revision"""
707 raise NotImplementedError
708
710 raise NotImplementedError
711
714 return first.when < second.when
715
718
720 return operator.attrgetter('when')
721
724 try:
725 return int(first.revision) < int(second.revision)
726 except (TypeError, ValueError):
727 return False
728
730 try:
731 int(revision)
732 return True
733 except:
734 return False
735
737 return operator.attrgetter('revision')
738