1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 import re
18 from twisted.python import log, failure
19 from twisted.spread import pb
20 from buildbot.process import buildstep
21 from buildbot.status.results import SUCCESS, WARNINGS, FAILURE
22 from buildbot.status.logfile import STDOUT, STDERR
23
24
25
26 from buildbot.process.properties import WithProperties
27 _hush_pyflakes = [WithProperties]
28 del _hush_pyflakes
29
31 """I run a single shell command on the buildslave. I return FAILURE if
32 the exit code of that command is non-zero, SUCCESS otherwise. To change
33 this behavior, override my .evaluateCommand method.
34
35 By default, a failure of this step will mark the whole build as FAILURE.
36 To override this, give me an argument of flunkOnFailure=False .
37
38 I create a single Log named 'log' which contains the output of the
39 command. To create additional summary Logs, override my .createSummary
40 method.
41
42 The shell command I run (a list of argv strings) can be provided in
43 several ways:
44 - a class-level .command attribute
45 - a command= parameter to my constructor (overrides .command)
46 - set explicitly with my .setCommand() method (overrides both)
47
48 @ivar command: a list of renderable objects (typically strings or
49 WithProperties instances). This will be used by start()
50 to create a RemoteShellCommand instance.
51
52 @ivar logfiles: a dict mapping log NAMEs to workdir-relative FILENAMEs
53 of their corresponding logfiles. The contents of the file
54 named FILENAME will be put into a LogFile named NAME, ina
55 something approximating real-time. (note that logfiles=
56 is actually handled by our parent class LoggingBuildStep)
57
58 @ivar lazylogfiles: Defaults to False. If True, logfiles will be tracked
59 `lazily', meaning they will only be added when and if
60 they are written to. Empty or nonexistent logfiles
61 will be omitted. (Also handled by class
62 LoggingBuildStep.)
63
64 """
65
66 name = "shell"
67 renderables = [ 'description', 'descriptionDone', 'slaveEnvironment', 'remote_kwargs', 'command' ]
68 description = None
69 descriptionDone = None
70 command = None
71
72
73
74
75
76
77 flunkOnFailure = True
78
79 - def __init__(self, workdir=None,
80 description=None, descriptionDone=None,
81 command=None,
82 usePTY="slave-config",
83 **kwargs):
118
123
126
128 rkw = self.remote_kwargs
129 rkw['workdir'] = rkw['workdir'] or workdir
130
132 """
133 Get the current notion of the workdir. Note that this may change
134 between instantiation of the step and C{start}, as it is based on the
135 build's default workdir, and may even be C{None} before that point.
136 """
137 return self.remote_kwargs['workdir']
138
141
143 """Return a list of short strings to describe this step, for the
144 status display. This uses the first few words of the shell command.
145 You can replace this by setting .description in your subclass, or by
146 overriding this method to describe the step better.
147
148 @type done: boolean
149 @param done: whether the command is complete or not, to improve the
150 way the command is described. C{done=False} is used
151 while the command is still running, so a single
152 imperfect-tense verb is appropriate ('compiling',
153 'testing', ...) C{done=True} is used when the command
154 has finished, and the default getText() method adds some
155 text, so a simple noun is appropriate ('compile',
156 'tests' ...)
157 """
158
159 try:
160 if done and self.descriptionDone is not None:
161 return self.descriptionDone
162 if self.description is not None:
163 return self.description
164
165
166
167 if not self.command:
168 return ["???"]
169
170 words = self.command
171 if isinstance(words, (str, unicode)):
172 words = words.split()
173 if len(words) < 1:
174 return ["???"]
175 if len(words) == 1:
176 return ["'%s'" % words[0]]
177 if len(words) == 2:
178 return ["'%s" % words[0], "%s'" % words[1]]
179 return ["'%s" % words[0], "%s" % words[1], "...'"]
180 except:
181 log.err(failure.Failure(), "Error describing step")
182 return ["???"]
183
185
186
187
188
189
190
191 slaveEnv = self.slaveEnvironment
192 if slaveEnv:
193 if cmd.args['env'] is None:
194 cmd.args['env'] = {}
195 fullSlaveEnv = slaveEnv.copy()
196 fullSlaveEnv.update(cmd.args['env'])
197 cmd.args['env'] = fullSlaveEnv
198
199
200
213
227
228
229
231 name = "treesize"
232 command = ["du", "-s", "-k", "."]
233 description = "measuring tree size"
234 descriptionDone = "tree size measured"
235 kib = None
236
243
250
251 - def getText(self, cmd, results):
252 if self.kib is not None:
253 return ["treesize", "%d KiB" % self.kib]
254 return ["treesize", "unknown"]
255
256
258 name = "setproperty"
259 renderables = [ 'property' ]
260
261 - def __init__(self, property=None, extract_fn=None, strip=True, **kwargs):
276
278 if self.property:
279 if cmd.rc != 0:
280 return
281 result = cmd.logs['stdio'].getText()
282 if self.strip: result = result.strip()
283 propname = self.property
284 self.setProperty(propname, result, "SetProperty Step")
285 self.property_changes[propname] = result
286 else:
287 log = cmd.logs['stdio']
288 new_props = self.extract_fn(cmd.rc,
289 ''.join(log.getChunks([STDOUT], onlyText=True)),
290 ''.join(log.getChunks([STDERR], onlyText=True)))
291 for k,v in new_props.items():
292 self.setProperty(k, v, "SetProperty Step")
293 self.property_changes = new_props
294
296 if self.property_changes:
297 props_set = [ "%s: %r" % (k,v)
298 for k,v in self.property_changes.items() ]
299 self.addCompleteLog('property changes', "\n".join(props_set))
300
301 - def getText(self, cmd, results):
302 if len(self.property_changes) > 1:
303 return [ "%d properties set" % len(self.property_changes) ]
304 elif len(self.property_changes) == 1:
305 return [ "property '%s' set" % self.property_changes.keys()[0] ]
306 else:
307
308 return ShellCommand.getText(self, cmd, results)
309
318
320 """
321 FileWriter class that just puts received data into a buffer.
322
323 Used to upload a file from slave for inline processing rather than
324 writing into a file on master.
325 """
328
331
334
336 renderables = [ 'suppressionFile' ]
337
338 warnCount = 0
339 warningPattern = '.*warning[: ].*'
340
341 directoryEnterPattern = (u"make.*: Entering directory "
342 u"[\u2019\"`'](.*)[\u2019'`\"]")
343 directoryLeavePattern = "make.*: Leaving directory"
344 suppressionFile = None
345
346 commentEmptyLineRe = re.compile(r"^\s*(\#.*)?$")
347 suppressionLineRe = re.compile(r"^\s*(.+?)\s*:\s*(.+?)\s*(?:[:]\s*([0-9]+)(?:-([0-9]+))?\s*)?$")
348
349 - def __init__(self,
350 warningPattern=None, warningExtractor=None, maxWarnCount=None,
351 directoryEnterPattern=None, directoryLeavePattern=None,
352 suppressionFile=None, **kwargs):
381
383 """
384 This method can be used to add patters of warnings that should
385 not be counted.
386
387 It takes a single argument, a list of patterns.
388
389 Each pattern is a 4-tuple (FILE-RE, WARN-RE, START, END).
390
391 FILE-RE is a regular expression (string or compiled regexp), or None.
392 If None, the pattern matches all files, else only files matching the
393 regexp. If directoryEnterPattern is specified in the class constructor,
394 matching is against the full path name, eg. src/main.c.
395
396 WARN-RE is similarly a regular expression matched against the
397 text of the warning, or None to match all warnings.
398
399 START and END form an inclusive line number range to match against. If
400 START is None, there is no lower bound, similarly if END is none there
401 is no upper bound."""
402
403 for fileRe, warnRe, start, end in suppressionList:
404 if fileRe != None and isinstance(fileRe, basestring):
405 fileRe = re.compile(fileRe)
406 if warnRe != None and isinstance(warnRe, basestring):
407 warnRe = re.compile(warnRe)
408 self.suppressions.append((fileRe, warnRe, start, end))
409
411 """
412 Extract warning text as the whole line.
413 No file names or line numbers."""
414 return (None, None, line)
415
417 """
418 Extract file name, line number, and warning text as groups (1,2,3)
419 of warningPattern match."""
420 file = match.group(1)
421 lineNo = match.group(2)
422 if lineNo != None:
423 lineNo = int(lineNo)
424 text = match.group(3)
425 return (file, lineNo, text)
426
428 if self.suppressions:
429 (file, lineNo, text) = self.warningExtractor(self, line, match)
430 lineNo = lineNo and int(lineNo)
431
432 if file != None and file != "" and self.directoryStack:
433 currentDirectory = '/'.join(self.directoryStack)
434 if currentDirectory != None and currentDirectory != "":
435 file = "%s/%s" % (currentDirectory, file)
436
437
438 for fileRe, warnRe, start, end in self.suppressions:
439 if not (file == None or fileRe == None or fileRe.match(file)):
440 continue
441 if not (warnRe == None or warnRe.search(text)):
442 continue
443 if not ((start == None and end == None) or
444 (lineNo != None and start <= lineNo and end >= lineNo)):
445 continue
446 return
447
448 warnings.append(line)
449 self.warnCount += 1
450
468
490
492 """
493 Match log lines against warningPattern.
494
495 Warnings are collected into another log for this step, and the
496 build-wide 'warnings-count' is updated."""
497
498 self.warnCount = 0
499
500
501
502 wre = self.warningPattern
503 if isinstance(wre, str):
504 wre = re.compile(wre)
505
506 directoryEnterRe = self.directoryEnterPattern
507 if (directoryEnterRe != None
508 and isinstance(directoryEnterRe, basestring)):
509 directoryEnterRe = re.compile(directoryEnterRe)
510
511 directoryLeaveRe = self.directoryLeavePattern
512 if (directoryLeaveRe != None
513 and isinstance(directoryLeaveRe, basestring)):
514 directoryLeaveRe = re.compile(directoryLeaveRe)
515
516
517
518
519 warnings = []
520
521
522 for line in log.getText().split("\n"):
523 if directoryEnterRe:
524 match = directoryEnterRe.search(line)
525 if match:
526 self.directoryStack.append(match.group(1))
527 continue
528 if (directoryLeaveRe and
529 self.directoryStack and
530 directoryLeaveRe.search(line)):
531 self.directoryStack.pop()
532 continue
533
534 match = wre.match(line)
535 if match:
536 self.maybeAddWarning(warnings, line, match)
537
538
539
540 if self.warnCount:
541 self.addCompleteLog("warnings (%d)" % self.warnCount,
542 "\n".join(warnings) + "\n")
543
544 warnings_stat = self.step_status.getStatistic('warnings', 0)
545 self.step_status.setStatistic('warnings', warnings_stat + self.warnCount)
546
547 try:
548 old_count = self.getProperty("warnings-count")
549 except KeyError:
550 old_count = 0
551 self.setProperty("warnings-count", old_count + self.warnCount, "WarningCountingShellCommand")
552
553
561
562
563 -class Compile(WarningCountingShellCommand):
571
572 -class Test(WarningCountingShellCommand):
615
617 command=["prove", "--lib", "lib", "-r", "t"]
618 total = 0
619
621
622 lines = map(
623 lambda line : line.replace('\r\n','').replace('\r','').replace('\n',''),
624 self.getLog('stdio').readlines()
625 )
626
627 total = 0
628 passed = 0
629 failed = 0
630 rc = SUCCESS
631 if cmd.rc > 0:
632 rc = FAILURE
633
634
635 if "Test Summary Report" in lines:
636 test_summary_report_index = lines.index("Test Summary Report")
637 del lines[0:test_summary_report_index + 2]
638
639 re_test_result = re.compile("^Result: (PASS|FAIL)$|Tests: \d+ Failed: (\d+)\)|Files=\d+, Tests=(\d+)")
640
641 mos = map(lambda line: re_test_result.search(line), lines)
642 test_result_lines = [mo.groups() for mo in mos if mo]
643
644 for line in test_result_lines:
645 if line[0] == 'FAIL':
646 rc = FAILURE
647
648 if line[1]:
649 failed += int(line[1])
650 if line[2]:
651 total = int(line[2])
652
653 else:
654 re_test_result = re.compile("^(All tests successful)|(\d+)/(\d+) subtests failed|Files=\d+, Tests=(\d+),")
655
656 mos = map(lambda line: re_test_result.search(line), lines)
657 test_result_lines = [mo.groups() for mo in mos if mo]
658
659 if test_result_lines:
660 test_result_line = test_result_lines[0]
661
662 success = test_result_line[0]
663
664 if success:
665 failed = 0
666
667 test_totals_line = test_result_lines[1]
668 total_str = test_totals_line[3]
669 else:
670 failed_str = test_result_line[1]
671 failed = int(failed_str)
672
673 total_str = test_result_line[2]
674
675 rc = FAILURE
676
677 total = int(total_str)
678
679 warnings = 0
680 if self.warningPattern:
681 wre = self.warningPattern
682 if isinstance(wre, str):
683 wre = re.compile(wre)
684
685 warnings = len([l for l in lines if wre.search(l)])
686
687
688
689
690 if rc == SUCCESS and warnings:
691 rc = WARNINGS
692
693 if total:
694 passed = total - failed
695
696 self.setTestResults(total=total, failed=failed, passed=passed,
697 warnings=warnings)
698
699 return rc
700