1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 import collections
17 import re
18 import warnings
19 import weakref
20 from buildbot import config, util
21 from buildbot.util import json, flatten
22 from buildbot.interfaces import IRenderable, IProperties
23 from twisted.internet import defer
24 from twisted.python.components import registerAdapter
25 from zope.interface import implements
28 """
29 I represent a set of properties that can be interpolated into various
30 strings in buildsteps.
31
32 @ivar properties: dictionary mapping property values to tuples
33 (value, source), where source is a string identifing the source
34 of the property.
35
36 Objects of this class can be read like a dictionary -- in this case,
37 only the property value is returned.
38
39 As a special case, a property value of None is returned as an empty
40 string when used as a mapping.
41 """
42
43 compare_attrs = ('properties',)
44 implements(IProperties)
45
47 """
48 @param kwargs: initial property values (for testing)
49 """
50 self.properties = {}
51
52
53 self.runtime = set()
54 self.build = None
55 if kwargs: self.update(kwargs, "TEST")
56
57 @classmethod
63
65 d = self.__dict__.copy()
66 d['build'] = None
67 return d
68
70 self.__dict__ = d
71 if not hasattr(self, 'runtime'):
72 self.runtime = set()
73
76
78 """Just get the value for this property."""
79 rv = self.properties[name][0]
80 return rv
81
84
87
89 """Return the properties as a sorted list of (name, value, source)"""
90 l = [ (k, v[0], v[1]) for k,v in self.properties.iteritems() ]
91 l.sort()
92 return l
93
95 """Return the properties as a simple key:value dictionary"""
96 return dict(self.properties)
97
99 return ('Properties(**' +
100 repr(dict((k,v[0]) for k,v in self.properties.iteritems())) +
101 ')')
102
103 - def update(self, dict, source, runtime=False):
104 """Update this object from a dictionary, with an explicit source specified."""
105 for k, v in dict.items():
106 self.setProperty(k, v, source, runtime=runtime)
107
112
114 """Update this object based on another object, but don't
115 include properties that were marked as runtime."""
116 for k,v in other.properties.iteritems():
117 if k not in other.runtime:
118 self.properties[k] = v
119
120
121
124
127
128 has_key = hasProperty
129
130 - def setProperty(self, name, value, source, runtime=False):
131 try:
132 json.dumps(value)
133 except TypeError:
134 warnings.warn(
135 "Non jsonable properties are not explicitly supported and" +
136 "will be explicitly disallowed in a future version.",
137 DeprecationWarning, stacklevel=2)
138
139 self.properties[name] = (value, source)
140 if runtime:
141 self.runtime.add(name)
142
145
148
152
155 """
156 A mixin to add L{IProperties} methods to a class which does not implement
157 the interface, but which can be coerced to the interface via an adapter.
158
159 This is useful because L{IProperties} methods are often called on L{Build}
160 and L{BuildStatus} objects without first coercing them.
161
162 @ivar set_runtime_properties: the default value for the C{runtime}
163 parameter of L{setProperty}.
164 """
165
166 set_runtime_properties = False
167
171
175
176 has_key = hasProperty
177
178 - def setProperty(self, propname, value, source='Unknown', runtime=None):
185
188
192
196 """
197 Privately-used mapping object to implement WithProperties' substitutions,
198 including the rendering of None as ''.
199 """
200 colon_minus_re = re.compile(r"(.*):-(.*)")
201 colon_tilde_re = re.compile(r"(.*):~(.*)")
202 colon_plus_re = re.compile(r"(.*):\+(.*)")
207
209 properties = self.properties()
210 assert properties is not None
211
212 def colon_minus(mo):
213
214
215 prop, repl = mo.group(1,2)
216 if prop in self.temp_vals:
217 return self.temp_vals[prop]
218 elif properties.has_key(prop):
219 return properties[prop]
220 else:
221 return repl
222
223 def colon_tilde(mo):
224
225
226 prop, repl = mo.group(1,2)
227 if prop in self.temp_vals and self.temp_vals[prop]:
228 return self.temp_vals[prop]
229 elif properties.has_key(prop) and properties[prop]:
230 return properties[prop]
231 else:
232 return repl
233
234 def colon_plus(mo):
235
236
237 prop, repl = mo.group(1,2)
238 if properties.has_key(prop) or prop in self.temp_vals:
239 return repl
240 else:
241 return ''
242
243 for regexp, fn in [
244 ( self.colon_minus_re, colon_minus ),
245 ( self.colon_tilde_re, colon_tilde ),
246 ( self.colon_plus_re, colon_plus ),
247 ]:
248 mo = regexp.match(key)
249 if mo:
250 rv = fn(mo)
251 break
252 else:
253
254
255 if key in self.temp_vals:
256 rv = self.temp_vals[key]
257 else:
258 rv = properties[key]
259
260
261 if rv is None: rv = ''
262 return rv
263
265 'Add a temporary value (to support keyword arguments to WithProperties)'
266 self.temp_vals[key] = val
267
269 """
270 This is a marker class, used fairly widely to indicate that we
271 want to interpolate build properties.
272 """
273
274 implements(IRenderable)
275 compare_attrs = ('fmtstring', 'args', 'lambda_subs')
276
277 - def __init__(self, fmtstring, *args, **lambda_subs):
278 self.fmtstring = fmtstring
279 self.args = args
280 if not self.args:
281 self.lambda_subs = lambda_subs
282 for key, val in self.lambda_subs.iteritems():
283 if not callable(val):
284 raise ValueError('Value for lambda substitution "%s" must be callable.' % key)
285 elif lambda_subs:
286 raise ValueError('WithProperties takes either positional or keyword substitutions, not both.')
287
289 pmap = _PropertyMap(build.getProperties())
290 if self.args:
291 strings = []
292 for name in self.args:
293 strings.append(pmap[name])
294 s = self.fmtstring % tuple(strings)
295 else:
296 for k,v in self.lambda_subs.iteritems():
297 pmap.add_temporary_value(k, v(build))
298 s = self.fmtstring % pmap
299 return s
300
301
302
303 _notHasKey = object()
304 -class _Lookup(util.ComparableMixin, object):
305 implements(IRenderable)
306
307 compare_attrs = ('value', 'index', 'default', 'defaultWhenFalse', 'hasKey', 'elideNoneAs')
308
309 - def __init__(self, value, index, default=None,
310 defaultWhenFalse=True, hasKey=_notHasKey,
311 elideNoneAs=None):
312 self.value = value
313 self.index = index
314 self.default = default
315 self.defaultWhenFalse = defaultWhenFalse
316 self.hasKey = hasKey
317 self.elideNoneAs = elideNoneAs
318
320 return '_Lookup(%r, %r%s%s%s%s)' % (
321 self.value,
322 self.index,
323 ', default=%r' % (self.default,)
324 if self.default is not None else '',
325 ', defaultWhenFalse=False'
326 if not self.defaultWhenFalse else '',
327 ', hasKey=%r' % (self.hasKey,)
328 if self.hasKey is not _notHasKey else '',
329 ', elideNoneAs=%r'% (self.elideNoneAs,)
330 if self.elideNoneAs is not None else '')
331
332
333 @defer.inlineCallbacks
354
357
358 dd = collections.defaultdict(str)
359 fmtstring % dd
360 return dd.keys()
361
367 _thePropertyDict = _PropertyDict()
382
383 -class _Lazy(util.ComparableMixin, object):
394
397 """
398 This is a marker class, used fairly widely to indicate that we
399 want to interpolate build properties.
400 """
401
402 implements(IRenderable)
403 compare_attrs = ('fmtstring', 'args', 'kwargs')
404
405 identifier_re = re.compile('^[\w-]*$')
406
407 - def __init__(self, fmtstring, *args, **kwargs):
408 self.fmtstring = fmtstring
409 self.args = args
410 self.kwargs = kwargs
411 if self.args and self.kwargs:
412 config.error("Interpolate takes either positional or keyword "
413 "substitutions, not both.")
414 if not self.args:
415 self.interpolations = {}
416 self._parse(fmtstring)
417
418
420 if self.args:
421 return 'Interpolate(%r, *%r)' % (self.fmtstring, self.args)
422 elif self.kwargs:
423 return 'Interpolate(%r, **%r)' % (self.fmtstring, self.kwargs)
424 else:
425 return 'Interpolate(%r)' % (self.fmtstring,)
426
427 @staticmethod
429 try:
430 prop, repl = arg.split(":", 1)
431 except ValueError:
432 prop, repl = arg, None
433 if not Interpolate.identifier_re.match(prop):
434 config.error("Property name must be alphanumeric for prop Interpolation '%s'" % arg)
435 prop = repl = None
436 return _thePropertyDict, prop, repl
437
438 @staticmethod
440
441 try:
442 codebase, attr, repl = arg.split(":", 2)
443 except ValueError:
444 try:
445 codebase, attr = arg.split(":",1)
446 repl = None
447 except ValueError:
448 config.error("Must specify both codebase and attribute for src Interpolation '%s'" % arg)
449 return {}, None, None
450
451 if not Interpolate.identifier_re.match(codebase):
452 config.error("Codebase must be alphanumeric for src Interpolation '%s'" % arg)
453 codebase = attr = repl = None
454 if not Interpolate.identifier_re.match(attr):
455 config.error("Attribute must be alphanumeric for src Interpolation '%s'" % arg)
456 codebase = attr = repl = None
457 return _SourceStampDict(codebase), attr, repl
458
460 try:
461 kw, repl = arg.split(":", 1)
462 except ValueError:
463 kw, repl = arg, None
464 if not Interpolate.identifier_re.match(kw):
465 config.error("Keyword must be alphanumeric for kw Interpolation '%s'" % arg)
466 kw = repl = None
467 return _Lazy(self.kwargs), kw, repl
468
470 try:
471 key, arg = fmt.split(":", 1)
472 except ValueError:
473 config.error("invalid Interpolate substitution without selector '%s'" % fmt)
474 return
475
476 fn = getattr(self, "_parse_" + key, None)
477 if not fn:
478 config.error("invalid Interpolate selector '%s'" % key)
479 return None
480 else:
481 return fn(arg)
482
483 @staticmethod
485 parenCount = 0
486 for i in range(0, len(arg)):
487 if arg[i] == "(":
488 parenCount += 1
489 if arg[i] == ")":
490 parenCount -= 1
491 if parenCount < 0:
492 raise ValueError
493 if parenCount == 0 and arg[i] == delim:
494 return arg[0:i], arg[i+1:]
495 return arg
496
498 return _Lookup(d, kw,
499 default=Interpolate(repl, **self.kwargs),
500 defaultWhenFalse=False,
501 elideNoneAs='')
502
504 return _Lookup(d, kw,
505 default=Interpolate(repl, **self.kwargs),
506 defaultWhenFalse=True,
507 elideNoneAs='')
508
510 return _Lookup(d, kw,
511 hasKey=Interpolate(repl, **self.kwargs),
512 default='',
513 defaultWhenFalse=False,
514 elideNoneAs='')
515
517 delim = repl[0]
518 if delim == '(':
519 config.error("invalid Interpolate ternary delimiter '('")
520 return None
521 try:
522 truePart, falsePart = self._splitBalancedParen(delim, repl[1:])
523 except ValueError:
524 config.error("invalid Interpolate ternary expression '%s' with delimiter '%s'" % (repl[1:], repl[0]))
525 return None
526 return _Lookup(d, kw,
527 hasKey=Interpolate(truePart, **self.kwargs),
528 default=Interpolate(falsePart, **self.kwargs),
529 defaultWhenFalse=defaultWhenFalse,
530 elideNoneAs='')
531
533 return self._parseColon_ternary(d, kw, repl, defaultWhenFalse=True)
534
536 keys = _getInterpolationList(fmtstring)
537 for key in keys:
538 if not self.interpolations.has_key(key):
539 d, kw, repl = self._parseSubstitution(key)
540 if repl is None:
541 repl = '-'
542 for pattern, fn in [
543 ( "-", self._parseColon_minus ),
544 ( "~", self._parseColon_tilde ),
545 ( "+", self._parseColon_plus ),
546 ( "?", self._parseColon_ternary ),
547 ( "#?", self._parseColon_ternary_hash )
548 ]:
549 junk, matches, tail = repl.partition(pattern)
550 if not junk and matches:
551 self.interpolations[key] = fn(d, kw, tail)
552 break
553 if not self.interpolations.has_key(key):
554 config.error("invalid Interpolate default type '%s'" % repl[0])
555
557 props = props.getProperties()
558 if self.args:
559 d = props.render(self.args)
560 d.addCallback(lambda args:
561 self.fmtstring % tuple(args))
562 return d
563 else:
564 d = props.render(self.interpolations)
565 d.addCallback(lambda res:
566 self.fmtstring % res)
567 return d
568
570 """
571 An instance of this class renders a property of a build.
572 """
573
574 implements(IRenderable)
575
576 compare_attrs = ('key','default', 'defaultWhenFalse')
577
578 - def __init__(self, key, default=None, defaultWhenFalse=True):
579 """
580 @param key: Property to render.
581 @param default: Value to use if property isn't set.
582 @param defaultWhenFalse: When true (default), use default value
583 if property evaluates to False. Otherwise, use default value
584 only when property isn't set.
585 """
586 self.key = key
587 self.default = default
588 self.defaultWhenFalse = defaultWhenFalse
589
591 if self.defaultWhenFalse:
592 d = props.render(props.getProperty(self.key))
593 @d.addCallback
594 def checkDefault(rv):
595 if rv:
596 return rv
597 else:
598 return props.render(self.default)
599 return d
600 else:
601 if props.hasProperty(self.key):
602 return props.render(props.getProperty(self.key))
603 else:
604 return props.render(self.default)
605
607 """
608 An instance of this class flattens all nested lists in a list
609 """
610 implements(IRenderable)
611
612 compare_attrs = ('nestedlist')
613
614 - def __init__(self, nestedlist, types=(list, tuple)):
615 """
616 @param nestedlist: a list of values to render
617 @param types: only flatten these types. defaults to (list, tuple)
618 """
619 self.nestedlist = nestedlist
620 self.types = types
621
623 d = props.render(self.nestedlist)
624 def flat(r):
625 return flatten(r, self.types)
626 d.addCallback(flat)
627 return d
628
629 -class _Renderer(util.ComparableMixin, object):
639
642
644 """
645 Default IRenderable adaptor. Calls .getRenderingFor if availble, otherwise
646 returns argument unchanged.
647 """
648
649 implements(IRenderable)
650
656
659
660 registerAdapter(_DefaultRenderer, object, IRenderable)
664 """
665 List IRenderable adaptor. Maps Build.render over the list.
666 """
667
668 implements(IRenderable)
669
672
675
676 registerAdapter(_ListRenderer, list, IRenderable)
680 """
681 Tuple IRenderable adaptor. Maps Build.render over the tuple.
682 """
683
684 implements(IRenderable)
685
688
693
694 registerAdapter(_TupleRenderer, tuple, IRenderable)
698 """
699 Dict IRenderable adaptor. Maps Build.render over the keya and values in the dict.
700 """
701
702 implements(IRenderable)
703
705 self.value = _ListRenderer([ _TupleRenderer((k,v)) for k,v in value.iteritems() ])
706
711
712 registerAdapter(_DictRenderer, dict, IRenderable)
713