buildbot.status.web.base

1 2 import urlparse, urllib, time, re 3 import os, cgi, sys, locale 4 import jinja2 5 from zope.interface import Interface 6 from twisted.web import resource, static 7 from twisted.python import log 8 from buildbot.status import builder 9 from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, SKIPPED, EXCEPTION, RETRY 10 from buildbot import version, util 11 from buildbot.process.properties import Properties 12

13 -class ITopBox(Interface):

14 """I represent a box in the top row of the waterfall display: the one 15 which shows the status of the last build for each builder."""

16 - def getBox(self, request):

17 """Return a Box instance, which can produce a <td> cell. 18 """

19

20 -class ICurrentBox(Interface):

21 """I represent the 'current activity' box, just above the builder name."""

22 - def getBox(self, status):

23 """Return a Box instance, which can produce a <td> cell. 24 """

25

26 -class IBox(Interface):

27 """I represent a box in the waterfall display."""

28 - def getBox(self, request):

29 """Return a Box instance, which wraps an Event and can produce a <td> 30 cell. 31 """

32

33 -class IHTMLLog(Interface):

34 pass

35 36 css_classes = {SUCCESS: "success", 37 WARNINGS: "warnings", 38 FAILURE: "failure", 39 SKIPPED: "skipped", 40 EXCEPTION: "exception", 41 RETRY: "retry", 42 None: "", 43 } 44 45

46 -def getAndCheckProperties(req):

47 """ 48 Fetch custom build properties from the HTTP request of a "Force build" or 49 "Resubmit build" HTML form. 50 Check the names for valid strings, and return None if a problem is found. 51 Return a new Properties object containing each property found in req. 52 """ 53 properties = Properties() 54 for i in (1,2,3): 55 pname = req.args.get("property%dname" % i, [""])[0] 56 pvalue = req.args.get("property%dvalue" % i, [""])[0] 57 if pname and pvalue: 58 if not re.match(r'^[\w\.\-\/\~:]*$', pname) \ 59 or not re.match(r'^[\w\.\-\/\~:]*$', pvalue): 60 log.msg("bad property name='%s', value='%s'" % (pname, pvalue)) 61 return None 62 properties.setProperty(pname, pvalue, "Force Build Form") 63 return properties

64

65 -def build_get_class(b):

66 """ 67 Return the class to use for a finished build or buildstep, 68 based on the result. 69 """ 70 # FIXME: this getResults duplicity might need to be fixed 71 result = b.getResults() 72 #print "THOMAS: result for b %r: %r" % (b, result) 73 if isinstance(b, builder.BuildStatus): 74 result = b.getResults() 75 elif isinstance(b, builder.BuildStepStatus): 76 result = b.getResults()[0] 77 # after forcing a build, b.getResults() returns ((None, []), []), ugh 78 if isinstance(result, tuple): 79 result = result[0] 80 else: 81 raise TypeError, "%r is not a BuildStatus or BuildStepStatus" % b 82 83 if result == None: 84 # FIXME: this happens when a buildstep is running ? 85 return "running" 86 return builder.Results[result]

87

88 -def path_to_root(request):

89 # /waterfall : ['waterfall'] -> '' 90 # /somewhere/lower : ['somewhere', 'lower'] -> '../' 91 # /somewhere/indexy/ : ['somewhere', 'indexy', ''] -> '../../' 92 # / : [] -> '' 93 if request.prepath: 94 segs = len(request.prepath) - 1 95 else: 96 segs = 0 97 root = "../" * segs 98 return root

99

100 -def path_to_authfail(request):

101 return path_to_root(request) + "/authfail"

102

103 -def path_to_builder(request, builderstatus):

104 return (path_to_root(request) + 105 "builders/" + 106 urllib.quote(builderstatus.getName(), safe=''))

107

108 -def path_to_build(request, buildstatus):

109 return (path_to_builder(request, buildstatus.getBuilder()) + 110 "/builds/%d" % buildstatus.getNumber())

111

112 -def path_to_step(request, stepstatus):

113 return (path_to_build(request, stepstatus.getBuild()) + 114 "/steps/%s" % urllib.quote(stepstatus.getName(), safe=''))

115

116 -def path_to_slave(request, slave):

117 return (path_to_root(request) + 118 "buildslaves/" + 119 urllib.quote(slave.getName(), safe=''))

120

121 -def path_to_change(request, change):

122 return (path_to_root(request) + 123 "changes/%s" % change.number)

124

125 -class Box:

126 # a Box wraps an Event. The Box has HTML <td> parameters that Events 127 # lack, and it has a base URL to which each File's name is relative. 128 # Events don't know about HTML. 129 spacer = False

130 - def __init__(self, text=[], class_=None, urlbase=None, 131 **parms):

132 self.text = text 133 self.class_ = class_ 134 self.urlbase = urlbase 135 self.show_idle = 0 136 if parms.has_key('show_idle'): 137 del parms['show_idle'] 138 self.show_idle = 1 139 140 self.parms = parms

141 # parms is a dict of HTML parameters for the <td> element that will 142 # represent this Event in the waterfall display. 143

144 - def td(self, **props):

145 props.update(self.parms) 146 text = self.text 147 if not text and self.show_idle: 148 text = ["[idle]"] 149 props['class'] = self.class_ 150 props['text'] = text; 151 return props

152 153

154 -class ContextMixin(object):

155 - def getContext(self, request):

156 status = self.getStatus(request) 157 rootpath = path_to_root(request) 158 locale_enc = locale.getdefaultlocale()[1] 159 if locale_enc is not None: 160 locale_tz = unicode(time.tzname[time.localtime()[-1]], locale_enc) 161 else: 162 locale_tz = unicode(time.tzname[time.localtime()[-1]]) 163 return dict(project_url = status.getProjectURL(), 164 project_name = status.getProjectName(), 165 stylesheet = rootpath + 'default.css', 166 path_to_root = rootpath, 167 version = version, 168 time = time.strftime("%a %d %b %Y %H:%M:%S", 169 time.localtime(util.now())), 170 tz = locale_tz, 171 metatags = [], 172 title = self.getTitle(request), 173 welcomeurl = rootpath)

174

175 - def getStatus(self, request):

176 return request.site.buildbot_service.getStatus()

177

178 - def getTitle(self, request):

179 return self.title

180

181 -class HtmlResource(resource.Resource, ContextMixin):

182 # this is a cheap sort of template thingy 183 contentType = "text/html; charset=utf-8" 184 title = "Buildbot" 185 addSlash = False # adapted from Nevow 186

187 - def getChild(self, path, request):

188 if self.addSlash and path == "" and len(request.postpath) == 0: 189 return self 190 return resource.Resource.getChild(self, path, request)

191

192 - def render(self, request):

193 # tell the WebStatus about the HTTPChannel that got opened, so they 194 # can close it if we get reconfigured and the WebStatus goes away. 195 # They keep a weakref to this, since chances are good that it will be 196 # closed by the browser or by us before we get reconfigured. See 197 # ticket #102 for details. 198 if hasattr(request, "channel"): 199 # web.distrib.Request has no .channel 200 request.site.buildbot_service.registerChannel(request.channel) 201 202 # Our pages no longer require that their URL end in a slash. Instead, 203 # they all use request.childLink() or some equivalent which takes the 204 # last path component into account. This clause is left here for 205 # historical and educational purposes. 206 if False and self.addSlash and request.prepath[-1] != '': 207 # this is intended to behave like request.URLPath().child('') 208 # but we need a relative URL, since we might be living behind a 209 # reverse proxy 210 # 211 # note that the Location: header (as used in redirects) are 212 # required to have absolute URIs, and my attempt to handle 213 # reverse-proxies gracefully violates rfc2616. This frequently 214 # works, but single-component paths sometimes break. The best 215 # strategy is to avoid these redirects whenever possible by using 216 # HREFs with trailing slashes, and only use the redirects for 217 # manually entered URLs. 218 url = request.prePathURL() 219 scheme, netloc, path, query, fragment = urlparse.urlsplit(url) 220 new_url = request.prepath[-1] + "/" 221 if query: 222 new_url += "?" + query 223 request.redirect(new_url) 224 return '' 225 226 ctx = self.getContext(request) 227 228 data = self.content(request, ctx) 229 if isinstance(data, unicode): 230 data = data.encode("utf-8") 231 request.setHeader("content-type", self.contentType) 232 if request.method == "HEAD": 233 request.setHeader("content-length", len(data)) 234 return '' 235 return data

236

237 - def getAuthz(self, request):

238 return request.site.buildbot_service.authz

239

240 - def getBuildmaster(self, request):

241 return request.site.buildbot_service.master

242 243

244 -class StaticHTML(HtmlResource):

245 - def __init__(self, body, title):

246 HtmlResource.__init__(self) 247 self.bodyHTML = body 248 self.title = title

249 - def content(self, request, cxt):

250 cxt['content'] = self.bodyHTML 251 cxt['title'] = self.title 252 template = request.site.buildbot_service.templates.get_template("empty.html") 253 return template.render(**cxt)

254 255 # DirectoryLister isn't available in Twisted-2.5.0, and isn't compatible with what 256 # we need until 9.0.0, so we just skip this particular feature. 257 have_DirectoryLister = False 258 if hasattr(static, 'DirectoryLister'): 259 have_DirectoryLister = True

260 - class DirectoryLister(static.DirectoryLister, ContextMixin):

261 """This variant of the static.DirectoryLister uses a template 262 for rendering.""" 263 264 title = 'BuildBot' 265

266 - def render(self, request):

267 cxt = self.getContext(request) 268 269 if self.dirs is None: 270 directory = os.listdir(self.path) 271 directory.sort() 272 else: 273 directory = self.dirs 274 275 dirs, files = self._getFilesAndDirectories(directory) 276 277 cxt['path'] = cgi.escape(urllib.unquote(request.uri)) 278 cxt['directories'] = dirs 279 cxt['files'] = files 280 template = request.site.buildbot_service.templates.get_template("directory.html") 281 data = template.render(**cxt) 282 if isinstance(data, unicode): 283 data = data.encode("utf-8") 284 return data

285

286 -class StaticFile(static.File):

287 """This class adds support for templated directory 288 views.""" 289

290 - def directoryListing(self):

291 if have_DirectoryLister: 292 return DirectoryLister(self.path, 293 self.listNames(), 294 self.contentTypes, 295 self.contentEncodings, 296 self.defaultType) 297 else: 298 return static.Data(""" 299 Directory Listings require Twisted-9.0.0 or later 300 """, "text/plain")

301 302 303 MINUTE = 60 304 HOUR = 60*MINUTE 305 DAY = 24*HOUR 306 WEEK = 7*DAY 307 MONTH = 30*DAY 308

309 -def plural(word, words, num):

310 if int(num) == 1: 311 return "%d %s" % (num, word) 312 else: 313 return "%d %s" % (num, words)

314

315 -def abbreviate_age(age):

316 if age <= 90: 317 return "%s ago" % plural("second", "seconds", age) 318 if age < 90*MINUTE: 319 return "about %s ago" % plural("minute", "minutes", age / MINUTE) 320 if age < DAY: 321 return "about %s ago" % plural("hour", "hours", age / HOUR) 322 if age < 2*WEEK: 323 return "about %s ago" % plural("day", "days", age / DAY) 324 if age < 2*MONTH: 325 return "about %s ago" % plural("week", "weeks", age / WEEK) 326 return "a long time ago"

327 328

329 -class BuildLineMixin:

330 LINE_TIME_FORMAT = "%b %d %H:%M" 331

332 - def get_line_values(self, req, build, include_builder=True):

333 ''' 334 Collect the data needed for each line display 335 ''' 336 builder_name = build.getBuilder().getName() 337 results = build.getResults() 338 text = build.getText() 339 try: 340 rev = build.getProperty("got_revision") 341 if rev is None: 342 rev = "??" 343 except KeyError: 344 rev = "??" 345 rev = str(rev) 346 css_class = css_classes.get(results, "") 347 348 if type(text) == list: 349 text = " ".join(text) 350 351 values = {'class': css_class, 352 'builder_name': builder_name, 353 'buildnum': build.getNumber(), 354 'results': css_class, 355 'text': " ".join(build.getText()), 356 'buildurl': path_to_build(req, build), 357 'builderurl': path_to_builder(req, build.getBuilder()), 358 'rev': rev, 359 'time': time.strftime(self.LINE_TIME_FORMAT, 360 time.localtime(build.getTimes()[0])), 361 'text': text, 362 'include_builder': include_builder 363 } 364 return values

365

366 -def map_branches(branches):

367 # when the query args say "trunk", present that to things like 368 # IBuilderStatus.generateFinishedBuilds as None, since that's the 369 # convention in use. But also include 'trunk', because some VC systems 370 # refer to it that way. In the long run we should clean this up better, 371 # maybe with Branch objects or something. 372 if "trunk" in branches: 373 return branches + [None] 374 return branches

375 376 377 # jinja utilities 378

379 -def createJinjaEnv(revlink=None, changecommentlink=None, 380 repositories=None, projects=None):

381 ''' Create a jinja environment changecommentlink is used to 382 render HTML in the WebStatus and for mail changes 383 384 @type changecommentlink: C{None}, tuple (2 or 3 strings), dict (string -> 2- or 3-tuple) or callable 385 @param changecommentlink: see changelinkfilter() 386 387 @type revlink: C{None}, format-string, dict (repository -> format string) or callable 388 @param revlink: see revlinkfilter() 389 390 @type repositories: C{None} or dict (string -> url) 391 @param repositories: an (optinal) mapping from repository identifiers 392 (as given by Change sources) to URLs. Is used to create a link 393 on every place where a repository is listed in the WebStatus. 394 395 @type projects: C{None} or dict (string -> url) 396 @param projects: similar to repositories, but for projects. 397 ''' 398 399 # See http://buildbot.net/trac/ticket/658 400 assert not hasattr(sys, "frozen"), 'Frozen config not supported with jinja (yet)' 401 402 default_loader = jinja2.PackageLoader('buildbot.status.web', 'templates') 403 root = os.path.join(os.getcwd(), 'templates') 404 loader = jinja2.ChoiceLoader([jinja2.FileSystemLoader(root), 405 default_loader]) 406 env = jinja2.Environment(loader=loader, 407 extensions=['jinja2.ext.i18n'], 408 trim_blocks=True, 409 undefined=AlmostStrictUndefined) 410 411 env.install_null_translations() # needed until we have a proper i18n backend 412 413 env.filters.update(dict( 414 urlencode = urllib.quote, 415 email = emailfilter, 416 user = userfilter, 417 shortrev = shortrevfilter(revlink, env), 418 revlink = revlinkfilter(revlink, env), 419 changecomment = changelinkfilter(changecommentlink), 420 repolink = dictlinkfilter(repositories), 421 projectlink = dictlinkfilter(projects) 422 )) 423 424 return env

425

426 -def emailfilter(value):

427 ''' Escape & obfuscate e-mail addresses 428 429 replacing @ with <span style="display:none> reportedly works well against web-spiders 430 and the next level is to use rot-13 (or something) and decode in javascript ''' 431 432 user = jinja2.escape(value) 433 obfuscator = jinja2.Markup('<span style="display:none">ohnoyoudont</span>@') 434 output = user.replace('@', obfuscator) 435 return output

436 437

438 -def userfilter(value):

439 ''' Hide e-mail address from user name when viewing changes 440 441 We still include the (obfuscated) e-mail so that we can show 442 it on mouse-over or similar etc 443 ''' 444 r = re.compile('(.*) +<(.*)>') 445 m = r.search(value) 446 if m: 447 user = jinja2.escape(m.group(1)) 448 email = emailfilter(m.group(2)) 449 return jinja2.Markup('<div class="user">%s<div class="email">%s</div></div>' % (user, email)) 450 else: 451 return emailfilter(value) # filter for emails here for safety

452

453 -def _revlinkcfg(replace, templates):

454 '''Helper function that returns suitable macros and functions 455 for building revision links depending on replacement mechanism 456 ''' 457 458 assert not replace or callable(replace) or isinstance(replace, dict) or \ 459 isinstance(replace, str) or isinstance(replace, unicode) 460 461 if not replace: 462 return lambda rev, repo: None 463 else: 464 if callable(replace): 465 return lambda rev, repo: replace(rev, repo) 466 elif isinstance(replace, dict): # TODO: test for [] instead 467 def filter(rev, repo): 468 url = replace.get(repo) 469 if url: 470 return url % urllib.quote(rev) 471 else: 472 return None

473 474 return filter 475 else: 476 return lambda rev, repo: replace % urllib.quote(rev) 477 478 assert False, '_replace has a bad type, but we should never get here' 479 480

481 -def _revlinkmacros(replace, templates):

482 '''return macros for use with revision links, depending 483 on whether revlinks are configured or not''' 484 485 macros = templates.get_template("revmacros.html").module 486 487 if not replace: 488 id = macros.id 489 short = macros.shorten 490 else: 491 id = macros.id_replace 492 short = macros.shorten_replace 493 494 return (id, short) 495 496

497 -def shortrevfilter(replace, templates):

498 ''' Returns a function which shortens the revisison string 499 to 12-chars (chosen as this is the Mercurial short-id length) 500 and add link if replacement string is set. 501 502 (The full id is still visible in HTML, for mouse-over events etc.) 503 504 @param replace: see revlinkfilter() 505 @param templates: a jinja2 environment 506 ''' 507 508 url_f = _revlinkcfg(replace, templates) 509 510 def filter(rev, repo): 511 if not rev: 512 return u'' 513 514 id_html, short_html = _revlinkmacros(replace, templates) 515 rev = unicode(rev) 516 url = url_f(rev, repo) 517 rev = jinja2.escape(rev) 518 shortrev = rev[:12] # TODO: customize this depending on vc type 519 520 if shortrev == rev: 521 if url: 522 return id_html(rev=rev, url=url) 523 else: 524 return rev 525 else: 526 if url: 527 return short_html(short=shortrev, rev=rev, url=url) 528 else: 529 return shortrev + '...'

530 531 return filter 532 533

534 -def revlinkfilter(replace, templates):

535 ''' Returns a function which adds an url link to a 536 revision identifiers. 537 538 Takes same params as shortrevfilter() 539 540 @param replace: either a python format string with an %s, 541 or a dict mapping repositories to format strings, 542 or a callable taking (revision, repository) arguments 543 and return an URL (or None, if no URL is available), 544 or None, in which case revisions do not get decorated 545 with links 546 547 @param templates: a jinja2 environment 548 ''' 549 550 url_f = _revlinkcfg(replace, templates) 551 552 def filter(rev, repo): 553 if not rev: 554 return u'' 555 556 rev = unicode(rev) 557 url = url_f(rev, repo) 558 if url: 559 id_html, _ = _revlinkmacros(replace, templates) 560 return id_html(rev=rev, url=url) 561 else: 562 return jinja2.escape(rev)

563 564 return filter 565 566

567 -def changelinkfilter(changelink):

568 ''' Returns function that does regex search/replace in 569 comments to add links to bug ids and similar. 570 571 @param changelink: 572 Either C{None} 573 or: a tuple (2 or 3 elements) 574 1. a regex to match what we look for 575 2. an url with regex refs (\g<0>, \1, \2, etc) that becomes the 'href' attribute 576 3. (optional) an title string with regex ref regex 577 or: a dict mapping projects to above tuples 578 (no links will be added if the project isn't found) 579 or: a callable taking (changehtml, project) args 580 (where the changetext is HTML escaped in the 581 form of a jinja2.Markup instance) and 582 returning another jinja2.Markup instance with 583 the same change text plus any HTML tags added to it. 584 ''' 585 586 assert not changelink or isinstance(changelink, dict) or \ 587 isinstance(changelink, tuple) or callable(changelink) 588 589 def replace_from_tuple(t): 590 search, url_replace = t[:2] 591 if len(t) == 3: 592 title_replace = ' title="%s"' % t[2] 593 else: 594 title_replace = '' 595 596 search_re = re.compile(search) 597 link_replace_re = jinja2.Markup(r'<a href="%s"%s>\g<0></a>' % (url_replace, title_replace)) 598 599 def filter(text, project): 600 text = jinja2.escape(text) 601 html = search_re.sub(link_replace_re, text) 602 return html

603 604 return filter 605 606 if not changelink: 607 return lambda text, project: jinja2.escape(text) 608 609 elif isinstance(changelink, dict): 610 def dict_filter(text, project): 611 # TODO: Optimize and cache return value from replace_from_tuple so 612 # we only compile regex once per project, not per view 613 614 t = changelink.get(project) 615 if t: 616 return replace_from_tuple(t)(text, project) 617 else: 618 return jinja2.escape(text) 619 620 return dict_filter 621 622 elif isinstance(changelink, tuple): 623 return replace_from_tuple(changelink) 624 625 elif callable(changelink): 626 def callable_filter(text, project): 627 text = jinja2.escape(text) 628 return changelink(text, project) 629 630 return callable_filter 631 632 assert False, 'changelink has unsupported type, but that is checked before' 633 634

635 -def dictlinkfilter(links):

636 '''A filter that encloses the given value in a link tag 637 given that the value exists in the dictionary''' 638 639 assert not links or callable(links) or isinstance(links, dict) 640 641 if not links: 642 return jinja2.escape 643 644 def filter(key): 645 if callable(links): 646 url = links(key) 647 else: 648 url = links.get(key) 649 650 safe_key = jinja2.escape(key) 651 652 if url: 653 return jinja2.Markup(r'<a href="%s">%s</a>' % (url, safe_key)) 654 else: 655 return safe_key

656 657 return filter 658

659 -class AlmostStrictUndefined(jinja2.StrictUndefined):

660 ''' An undefined that allows boolean testing but 661 fails properly on every other use. 662 663 Much better than the default Undefined, but not 664 fully as strict as StrictUndefined '''

665 - def __nonzero__(self):

666 return False

667

Source Code for Module buildbot.status.web.base