1
2
3 import re
4 from twisted.python import log
5 from twisted.spread import pb
6 from buildbot.process.buildstep import LoggingBuildStep, RemoteShellCommand
7 from buildbot.process.buildstep import RemoteCommand
8 from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, STDOUT, STDERR
9
10
11
12 from buildbot.process.properties import WithProperties
13 _hush_pyflakes = [WithProperties]
14 del _hush_pyflakes
15
17 """I run a single shell command on the buildslave. I return FAILURE if
18 the exit code of that command is non-zero, SUCCESS otherwise. To change
19 this behavior, override my .evaluateCommand method.
20
21 By default, a failure of this step will mark the whole build as FAILURE.
22 To override this, give me an argument of flunkOnFailure=False .
23
24 I create a single Log named 'log' which contains the output of the
25 command. To create additional summary Logs, override my .createSummary
26 method.
27
28 The shell command I run (a list of argv strings) can be provided in
29 several ways:
30 - a class-level .command attribute
31 - a command= parameter to my constructor (overrides .command)
32 - set explicitly with my .setCommand() method (overrides both)
33
34 @ivar command: a list of renderable objects (typically strings or
35 WithProperties instances). This will be used by start()
36 to create a RemoteShellCommand instance.
37
38 @ivar logfiles: a dict mapping log NAMEs to workdir-relative FILENAMEs
39 of their corresponding logfiles. The contents of the file
40 named FILENAME will be put into a LogFile named NAME, ina
41 something approximating real-time. (note that logfiles=
42 is actually handled by our parent class LoggingBuildStep)
43
44 @ivar lazylogfiles: Defaults to False. If True, logfiles will be tracked
45 `lazily', meaning they will only be added when and if
46 they are written to. Empty or nonexistent logfiles
47 will be omitted. (Also handled by class
48 LoggingBuildStep.)
49
50 """
51
52 name = "shell"
53 description = None
54 descriptionDone = None
55 command = None
56
57
58
59
60
61
62 flunkOnFailure = True
63
64 - def __init__(self, workdir=None,
65 description=None, descriptionDone=None,
66 command=None,
67 usePTY="slave-config",
68 **kwargs):
103
105 rkw = self.remote_kwargs
106 rkw['workdir'] = rkw['workdir'] or workdir
107
110
112 """Return a list of short strings to describe this step, for the
113 status display. This uses the first few words of the shell command.
114 You can replace this by setting .description in your subclass, or by
115 overriding this method to describe the step better.
116
117 @type done: boolean
118 @param done: whether the command is complete or not, to improve the
119 way the command is described. C{done=False} is used
120 while the command is still running, so a single
121 imperfect-tense verb is appropriate ('compiling',
122 'testing', ...) C{done=True} is used when the command
123 has finished, and the default getText() method adds some
124 text, so a simple noun is appropriate ('compile',
125 'tests' ...)
126 """
127
128 if done and self.descriptionDone is not None:
129 return list(self.descriptionDone)
130 if self.description is not None:
131 return list(self.description)
132
133 properties = self.build.getProperties()
134 words = self.command
135 if isinstance(words, (str, unicode)):
136 words = words.split()
137
138 words = properties.render(words)
139 if len(words) < 1:
140 return ["???"]
141 if len(words) == 1:
142 return ["'%s'" % words[0]]
143 if len(words) == 2:
144 return ["'%s" % words[0], "%s'" % words[1]]
145 return ["'%s" % words[0], "%s" % words[1], "...'"]
146
148
149
150
151
152
153
154 properties = self.build.getProperties()
155 slaveEnv = self.build.slaveEnvironment
156 if slaveEnv:
157 if cmd.args['env'] is None:
158 cmd.args['env'] = {}
159 fullSlaveEnv = slaveEnv.copy()
160 fullSlaveEnv.update(cmd.args['env'])
161 cmd.args['env'] = properties.render(fullSlaveEnv)
162
163
164
166 if not self.logfiles:
167 return
168 if not self.slaveVersionIsOlderThan("shell", "2.1"):
169 return
170
171
172
173
174 msg1 = ("Warning: buildslave %s is too old "
175 "to understand logfiles=, ignoring it."
176 % self.getSlaveName())
177 msg2 = "You will have to pull this logfile (%s) manually."
178 log.msg(msg1)
179 for logname,remotefilevalue in self.logfiles.items():
180 remotefilename = remotefilevalue
181
182 if type(remotefilevalue) == dict:
183 remotefilename = remotefilevalue['filename']
184
185 newlog = self.addLog(logname)
186 newlog.addHeader(msg1 + "\n")
187 newlog.addHeader(msg2 % remotefilename + "\n")
188 newlog.finish()
189
190 self.logfiles = {}
191
216
217
218
220 name = "treesize"
221 command = ["du", "-s", "-k", "."]
222 kib = None
223
225 out = cmd.logs['stdio'].getText()
226 m = re.search(r'^(\d+)', out)
227 if m:
228 self.kib = int(m.group(1))
229 self.setProperty("tree-size-KiB", self.kib, "treesize")
230
237
238 - def getText(self, cmd, results):
239 if self.kib is not None:
240 return ["treesize", "%d KiB" % self.kib]
241 return ["treesize", "unknown"]
242
244 name = "setproperty"
245
247 self.property = None
248 self.extract_fn = None
249 self.strip = True
250
251 if kwargs.has_key('property'):
252 self.property = kwargs['property']
253 del kwargs['property']
254 if kwargs.has_key('extract_fn'):
255 self.extract_fn = kwargs['extract_fn']
256 del kwargs['extract_fn']
257 if kwargs.has_key('strip'):
258 self.strip = kwargs['strip']
259 del kwargs['strip']
260
261 ShellCommand.__init__(self, **kwargs)
262
263 self.addFactoryArguments(property=self.property)
264 self.addFactoryArguments(extract_fn=self.extract_fn)
265 self.addFactoryArguments(strip=self.strip)
266
267 assert self.property or self.extract_fn, \
268 "SetProperty step needs either property= or extract_fn="
269
270 self.property_changes = {}
271
273 if self.property:
274 result = cmd.logs['stdio'].getText()
275 if self.strip: result = result.strip()
276 propname = self.build.getProperties().render(self.property)
277 self.setProperty(propname, result, "SetProperty Step")
278 self.property_changes[propname] = result
279 else:
280 log = cmd.logs['stdio']
281 new_props = self.extract_fn(cmd.rc,
282 ''.join(log.getChunks([STDOUT], onlyText=True)),
283 ''.join(log.getChunks([STDERR], onlyText=True)))
284 for k,v in new_props.items():
285 self.setProperty(k, v, "SetProperty Step")
286 self.property_changes = new_props
287
289 props_set = [ "%s: %r" % (k,v) for k,v in self.property_changes.items() ]
290 self.addCompleteLog('property changes', "\n".join(props_set))
291
292 - def getText(self, cmd, results):
293 if self.property_changes:
294 return [ "set props:" ] + self.property_changes.keys()
295 else:
296 return [ "no change" ]
297
306
308 """
309 FileWriter class that just puts received data into a buffer.
310
311 Used to upload a file from slave for inline processing rather than
312 writing into a file on master.
313 """
316
319
322
324 """
325 Remote command subclass used to run an internal file upload command on the
326 slave. We do not need any progress updates from such command, so override
327 remoteUpdate() with an empty method.
328 """
331
333 warnCount = 0
334 warningPattern = '.*warning[: ].*'
335
336 directoryEnterPattern = "make.*: Entering directory [\"`'](.*)['`\"]"
337 directoryLeavePattern = "make.*: Leaving directory"
338 suppressionFile = None
339
340 commentEmptyLineRe = re.compile(r"^\s*(\#.*)?$")
341 suppressionLineRe = re.compile(r"^\s*(.+?)\s*:\s*(.+?)\s*(?:[:]\s*([0-9]+)(?:-([0-9]+))?\s*)?$")
342
343 - def __init__(self, workdir=None,
344 warningPattern=None, warningExtractor=None,
345 directoryEnterPattern=None, directoryLeavePattern=None,
346 suppressionFile=None, **kwargs):
374
379
381 """
382 This method can be used to add patters of warnings that should
383 not be counted.
384
385 It takes a single argument, a list of patterns.
386
387 Each pattern is a 4-tuple (FILE-RE, WARN-RE, START, END).
388
389 FILE-RE is a regular expression (string or compiled regexp), or None.
390 If None, the pattern matches all files, else only files matching the
391 regexp. If directoryEnterPattern is specified in the class constructor,
392 matching is against the full path name, eg. src/main.c.
393
394 WARN-RE is similarly a regular expression matched against the
395 text of the warning, or None to match all warnings.
396
397 START and END form an inclusive line number range to match against. If
398 START is None, there is no lower bound, similarly if END is none there
399 is no upper bound."""
400
401 for fileRe, warnRe, start, end in suppressionList:
402 if fileRe != None and isinstance(fileRe, str):
403 fileRe = re.compile(fileRe)
404 if warnRe != None and isinstance(warnRe, str):
405 warnRe = re.compile(warnRe)
406 self.suppressions.append((fileRe, warnRe, start, end))
407
409 """
410 Extract warning text as the whole line.
411 No file names or line numbers."""
412 return (None, None, line)
413
415 """
416 Extract file name, line number, and warning text as groups (1,2,3)
417 of warningPattern match."""
418 file = match.group(1)
419 lineNo = match.group(2)
420 if lineNo != None:
421 lineNo = int(lineNo)
422 text = match.group(3)
423 return (file, lineNo, text)
424
426 if self.suppressions:
427 (file, lineNo, text) = self.warningExtractor(self, line, match)
428
429 if file != None and file != "" and self.directoryStack:
430 currentDirectory = self.directoryStack[-1]
431 if currentDirectory != None and currentDirectory != "":
432 file = "%s/%s" % (currentDirectory, file)
433
434
435 for fileRe, warnRe, start, end in self.suppressions:
436 if ( (file == None or fileRe == None or fileRe.search(file)) and
437 (warnRe == None or warnRe.search(text)) and
438 lineNo != None and
439 (start == None or start <= lineNo) and
440 (end == None or end >= lineNo) ):
441 return
442
443 warnings.append(line)
444 self.warnCount += 1
445
470
492
494 self.warnCount = 0
495
496
497
498 if not self.warningPattern:
499 return
500
501 wre = self.warningPattern
502 if isinstance(wre, str):
503 wre = re.compile(wre)
504
505 directoryEnterRe = self.directoryEnterPattern
506 if directoryEnterRe != None and isinstance(directoryEnterRe, str):
507 directoryEnterRe = re.compile(directoryEnterRe)
508
509 directoryLeaveRe = self.directoryLeavePattern
510 if directoryLeaveRe != None and isinstance(directoryLeaveRe, str):
511 directoryLeaveRe = re.compile(directoryLeaveRe)
512
513
514
515
516 warnings = []
517
518
519 for line in log.getText().split("\n"):
520 if directoryEnterRe:
521 match = directoryEnterRe.search(line)
522 if match:
523 self.directoryStack.append(match.group(1))
524 if (directoryLeaveRe and
525 self.directoryStack and
526 directoryLeaveRe.search(line)):
527 self.directoryStack.pop()
528
529 match = wre.match(line)
530 if match:
531 self.maybeAddWarning(warnings, line, match)
532
533
534
535 if self.warnCount:
536 self.addCompleteLog("warnings", "\n".join(warnings) + "\n")
537
538 warnings_stat = self.step_status.getStatistic('warnings', 0)
539 self.step_status.setStatistic('warnings', warnings_stat + self.warnCount)
540
541 try:
542 old_count = self.getProperty("warnings-count")
543 except KeyError:
544 old_count = 0
545 self.setProperty("warnings-count", old_count + self.warnCount, "WarningCountingShellCommand")
546
547
554
555
556 -class Compile(WarningCountingShellCommand):
573
574 -class Test(WarningCountingShellCommand):
616
618 command=["prove", "--lib", "lib", "-r", "t"]
619 total = 0
620
622
623 lines = map(
624 lambda line : line.replace('\r\n','').replace('\r','').replace('\n',''),
625 self.getLog('stdio').readlines()
626 )
627
628 total = 0
629 passed = 0
630 failed = 0
631 rc = cmd.rc
632
633
634 try:
635 test_summary_report_index = lines.index("Test Summary Report")
636
637 del lines[0:test_summary_report_index + 2]
638
639 re_test_result = re.compile("^Result: (PASS|FAIL)$|Tests: \d+ Failed: (\d+)\)|Files=\d+, Tests=(\d+)")
640
641 mos = map(lambda line: re_test_result.search(line), lines)
642 test_result_lines = [mo.groups() for mo in mos if mo]
643
644 for line in test_result_lines:
645 if line[0] == 'PASS':
646 rc = SUCCESS
647 elif line[0] == 'FAIL':
648 rc = FAILURE
649 elif line[1]:
650 failed += int(line[1])
651 elif line[2]:
652 total = int(line[2])
653
654 except ValueError:
655 re_test_result = re.compile("^(All tests successful)|(\d+)/(\d+) subtests failed|Files=\d+, Tests=(\d+),")
656
657 mos = map(lambda line: re_test_result.search(line), lines)
658 test_result_lines = [mo.groups() for mo in mos if mo]
659
660 if test_result_lines:
661 test_result_line = test_result_lines[0]
662
663 success = test_result_line[0]
664
665 if success:
666 failed = 0
667
668 test_totals_line = test_result_lines[1]
669 total_str = test_totals_line[3]
670 rc = SUCCESS
671 else:
672 failed_str = test_result_line[1]
673 failed = int(failed_str)
674
675 total_str = test_result_line[2]
676
677 rc = FAILURE
678
679 total = int(total_str)
680
681 warnings = 0
682 if self.warningPattern:
683 wre = self.warningPattern
684 if isinstance(wre, str):
685 wre = re.compile(wre)
686
687 warnings = len([l for l in lines if wre.search(l)])
688
689
690
691
692 if rc == SUCCESS and warnings:
693 rc = WARNINGS
694
695 if total:
696 passed = total - failed
697
698 self.setTestResults(total=total, failed=failed, passed=passed,
699 warnings=warnings)
700
701 return rc
702