1
2
3 import re
4 from twisted.python import log
5 from twisted.spread import pb
6 from buildbot.process.buildstep import LoggingBuildStep, RemoteShellCommand
7 from buildbot.process.buildstep import RemoteCommand
8 from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, STDOUT, STDERR
9 from buildbot.interfaces import BuildSlaveTooOldError
10
11
12
13 from buildbot.process.properties import WithProperties
14 _hush_pyflakes = [WithProperties]
15 del _hush_pyflakes
16
18 """I run a single shell command on the buildslave. I return FAILURE if
19 the exit code of that command is non-zero, SUCCESS otherwise. To change
20 this behavior, override my .evaluateCommand method.
21
22 By default, a failure of this step will mark the whole build as FAILURE.
23 To override this, give me an argument of flunkOnFailure=False .
24
25 I create a single Log named 'log' which contains the output of the
26 command. To create additional summary Logs, override my .createSummary
27 method.
28
29 The shell command I run (a list of argv strings) can be provided in
30 several ways:
31 - a class-level .command attribute
32 - a command= parameter to my constructor (overrides .command)
33 - set explicitly with my .setCommand() method (overrides both)
34
35 @ivar command: a list of renderable objects (typically strings or
36 WithProperties instances). This will be used by start()
37 to create a RemoteShellCommand instance.
38
39 @ivar logfiles: a dict mapping log NAMEs to workdir-relative FILENAMEs
40 of their corresponding logfiles. The contents of the file
41 named FILENAME will be put into a LogFile named NAME, ina
42 something approximating real-time. (note that logfiles=
43 is actually handled by our parent class LoggingBuildStep)
44
45 @ivar lazylogfiles: Defaults to False. If True, logfiles will be tracked
46 `lazily', meaning they will only be added when and if
47 they are written to. Empty or nonexistent logfiles
48 will be omitted. (Also handled by class
49 LoggingBuildStep.)
50
51 """
52
53 name = "shell"
54 description = None
55 descriptionDone = None
56 command = None
57
58
59
60
61
62
63 flunkOnFailure = True
64
65 - def __init__(self, workdir=None,
66 description=None, descriptionDone=None,
67 command=None,
68 usePTY="slave-config",
69 **kwargs):
104
107
109 rkw = self.remote_kwargs
110 rkw['workdir'] = rkw['workdir'] or workdir
111
114
116 """Return a list of short strings to describe this step, for the
117 status display. This uses the first few words of the shell command.
118 You can replace this by setting .description in your subclass, or by
119 overriding this method to describe the step better.
120
121 @type done: boolean
122 @param done: whether the command is complete or not, to improve the
123 way the command is described. C{done=False} is used
124 while the command is still running, so a single
125 imperfect-tense verb is appropriate ('compiling',
126 'testing', ...) C{done=True} is used when the command
127 has finished, and the default getText() method adds some
128 text, so a simple noun is appropriate ('compile',
129 'tests' ...)
130 """
131
132 try:
133 if done and self.descriptionDone is not None:
134 return list(self.descriptionDone)
135 if self.description is not None:
136 return list(self.description)
137
138 properties = self.build.getProperties()
139 words = self.command
140 if isinstance(words, (str, unicode)):
141 words = words.split()
142
143 words = properties.render(words)
144 if len(words) < 1:
145 return ["???"]
146 if len(words) == 1:
147 return ["'%s'" % words[0]]
148 if len(words) == 2:
149 return ["'%s" % words[0], "%s'" % words[1]]
150 return ["'%s" % words[0], "%s" % words[1], "...'"]
151 except:
152 log.msg("Error describing step")
153 log.err()
154 return ["???"]
155
157
158
159
160
161
162
163 properties = self.build.getProperties()
164 slaveEnv = self.build.slaveEnvironment
165 if slaveEnv:
166 if cmd.args['env'] is None:
167 cmd.args['env'] = {}
168 fullSlaveEnv = slaveEnv.copy()
169 fullSlaveEnv.update(cmd.args['env'])
170 cmd.args['env'] = properties.render(fullSlaveEnv)
171
172
173
175 if not self.logfiles:
176 return
177 if not self.slaveVersionIsOlderThan("shell", "2.1"):
178 return
179
180
181
182
183 msg1 = ("Warning: buildslave %s is too old "
184 "to understand logfiles=, ignoring it."
185 % self.getSlaveName())
186 msg2 = "You will have to pull this logfile (%s) manually."
187 log.msg(msg1)
188 for logname,remotefilevalue in self.logfiles.items():
189 remotefilename = remotefilevalue
190
191 if type(remotefilevalue) == dict:
192 remotefilename = remotefilevalue['filename']
193
194 newlog = self.addLog(logname)
195 newlog.addHeader(msg1 + "\n")
196 newlog.addHeader(msg2 % remotefilename + "\n")
197 newlog.finish()
198
199 self.logfiles = {}
200
225
226
227
229 name = "treesize"
230 command = ["du", "-s", "-k", "."]
231 description = "measuring tree size"
232 descriptionDone = "tree size measured"
233 kib = None
234
237
239 out = cmd.logs['stdio'].getText()
240 m = re.search(r'^(\d+)', out)
241 if m:
242 self.kib = int(m.group(1))
243 self.setProperty("tree-size-KiB", self.kib, "treesize")
244
251
252 - def getText(self, cmd, results):
253 if self.kib is not None:
254 return ["treesize", "%d KiB" % self.kib]
255 return ["treesize", "unknown"]
256
258 name = "setproperty"
259
260 - def __init__(self, property=None, extract_fn=None, strip=True, **kwargs):
275
291
293 props_set = [ "%s: %r" % (k,v) for k,v in self.property_changes.items() ]
294 self.addCompleteLog('property changes', "\n".join(props_set))
295
296 - def getText(self, cmd, results):
297 if self.property_changes:
298 return [ "set props:" ] + self.property_changes.keys()
299 else:
300 return [ "no change" ]
301
310
312 """
313 FileWriter class that just puts received data into a buffer.
314
315 Used to upload a file from slave for inline processing rather than
316 writing into a file on master.
317 """
320
323
326
328 """
329 Remote command subclass used to run an internal file upload command on the
330 slave. We do not need any progress updates from such command, so override
331 remoteUpdate() with an empty method.
332 """
335
337 warnCount = 0
338 warningPattern = '.*warning[: ].*'
339
340 directoryEnterPattern = "make.*: Entering directory [\"`'](.*)['`\"]"
341 directoryLeavePattern = "make.*: Leaving directory"
342 suppressionFile = None
343
344 commentEmptyLineRe = re.compile(r"^\s*(\#.*)?$")
345 suppressionLineRe = re.compile(r"^\s*(.+?)\s*:\s*(.+?)\s*(?:[:]\s*([0-9]+)(?:-([0-9]+))?\s*)?$")
346
347 - def __init__(self, workdir=None,
348 warningPattern=None, warningExtractor=None,
349 directoryEnterPattern=None, directoryLeavePattern=None,
350 suppressionFile=None, **kwargs):
378
383
385 """
386 This method can be used to add patters of warnings that should
387 not be counted.
388
389 It takes a single argument, a list of patterns.
390
391 Each pattern is a 4-tuple (FILE-RE, WARN-RE, START, END).
392
393 FILE-RE is a regular expression (string or compiled regexp), or None.
394 If None, the pattern matches all files, else only files matching the
395 regexp. If directoryEnterPattern is specified in the class constructor,
396 matching is against the full path name, eg. src/main.c.
397
398 WARN-RE is similarly a regular expression matched against the
399 text of the warning, or None to match all warnings.
400
401 START and END form an inclusive line number range to match against. If
402 START is None, there is no lower bound, similarly if END is none there
403 is no upper bound."""
404
405 for fileRe, warnRe, start, end in suppressionList:
406 if fileRe != None and isinstance(fileRe, str):
407 fileRe = re.compile(fileRe)
408 if warnRe != None and isinstance(warnRe, str):
409 warnRe = re.compile(warnRe)
410 self.suppressions.append((fileRe, warnRe, start, end))
411
413 """
414 Extract warning text as the whole line.
415 No file names or line numbers."""
416 return (None, None, line)
417
419 """
420 Extract file name, line number, and warning text as groups (1,2,3)
421 of warningPattern match."""
422 file = match.group(1)
423 lineNo = match.group(2)
424 if lineNo != None:
425 lineNo = int(lineNo)
426 text = match.group(3)
427 return (file, lineNo, text)
428
430 if self.suppressions:
431 (file, lineNo, text) = self.warningExtractor(self, line, match)
432
433 if file != None and file != "" and self.directoryStack:
434 currentDirectory = self.directoryStack[-1]
435 if currentDirectory != None and currentDirectory != "":
436 file = "%s/%s" % (currentDirectory, file)
437
438
439 for fileRe, warnRe, start, end in self.suppressions:
440 if ( (file == None or fileRe == None or fileRe.search(file)) and
441 (warnRe == None or warnRe.search(text)) and
442 lineNo != None and
443 (start == None or start <= lineNo) and
444 (end == None or end >= lineNo) ):
445 return
446
447 warnings.append(line)
448 self.warnCount += 1
449
474
496
498 """
499 Match log lines against warningPattern.
500
501 Warnings are collected into another log for this step, and the
502 build-wide 'warnings-count' is updated."""
503
504 self.warnCount = 0
505
506
507
508 if not self.warningPattern:
509 return
510
511 wre = self.warningPattern
512 if isinstance(wre, str):
513 wre = re.compile(wre)
514
515 directoryEnterRe = self.directoryEnterPattern
516 if directoryEnterRe != None and isinstance(directoryEnterRe, str):
517 directoryEnterRe = re.compile(directoryEnterRe)
518
519 directoryLeaveRe = self.directoryLeavePattern
520 if directoryLeaveRe != None and isinstance(directoryLeaveRe, str):
521 directoryLeaveRe = re.compile(directoryLeaveRe)
522
523
524
525
526 warnings = []
527
528
529 for line in log.getText().split("\n"):
530 if directoryEnterRe:
531 match = directoryEnterRe.search(line)
532 if match:
533 self.directoryStack.append(match.group(1))
534 if (directoryLeaveRe and
535 self.directoryStack and
536 directoryLeaveRe.search(line)):
537 self.directoryStack.pop()
538
539 match = wre.match(line)
540 if match:
541 self.maybeAddWarning(warnings, line, match)
542
543
544
545 if self.warnCount:
546 self.addCompleteLog("warnings", "\n".join(warnings) + "\n")
547
548 warnings_stat = self.step_status.getStatistic('warnings', 0)
549 self.step_status.setStatistic('warnings', warnings_stat + self.warnCount)
550
551 try:
552 old_count = self.getProperty("warnings-count")
553 except KeyError:
554 old_count = 0
555 self.setProperty("warnings-count", old_count + self.warnCount, "WarningCountingShellCommand")
556
557
564
565
566 -class Compile(WarningCountingShellCommand):
583
584 -class Test(WarningCountingShellCommand):
626
628 command=["prove", "--lib", "lib", "-r", "t"]
629 total = 0
630
632
633 lines = map(
634 lambda line : line.replace('\r\n','').replace('\r','').replace('\n',''),
635 self.getLog('stdio').readlines()
636 )
637
638 total = 0
639 passed = 0
640 failed = 0
641 rc = cmd.rc
642
643
644 try:
645 test_summary_report_index = lines.index("Test Summary Report")
646
647 del lines[0:test_summary_report_index + 2]
648
649 re_test_result = re.compile("^Result: (PASS|FAIL)$|Tests: \d+ Failed: (\d+)\)|Files=\d+, Tests=(\d+)")
650
651 mos = map(lambda line: re_test_result.search(line), lines)
652 test_result_lines = [mo.groups() for mo in mos if mo]
653
654 for line in test_result_lines:
655 if line[0] == 'PASS':
656 rc = SUCCESS
657 elif line[0] == 'FAIL':
658 rc = FAILURE
659 elif line[1]:
660 failed += int(line[1])
661 elif line[2]:
662 total = int(line[2])
663
664 except ValueError:
665 re_test_result = re.compile("^(All tests successful)|(\d+)/(\d+) subtests failed|Files=\d+, Tests=(\d+),")
666
667 mos = map(lambda line: re_test_result.search(line), lines)
668 test_result_lines = [mo.groups() for mo in mos if mo]
669
670 if test_result_lines:
671 test_result_line = test_result_lines[0]
672
673 success = test_result_line[0]
674
675 if success:
676 failed = 0
677
678 test_totals_line = test_result_lines[1]
679 total_str = test_totals_line[3]
680 rc = SUCCESS
681 else:
682 failed_str = test_result_line[1]
683 failed = int(failed_str)
684
685 total_str = test_result_line[2]
686
687 rc = FAILURE
688
689 total = int(total_str)
690
691 warnings = 0
692 if self.warningPattern:
693 wre = self.warningPattern
694 if isinstance(wre, str):
695 wre = re.compile(wre)
696
697 warnings = len([l for l in lines if wre.search(l)])
698
699
700
701
702 if rc == SUCCESS and warnings:
703 rc = WARNINGS
704
705 if total:
706 passed = total - failed
707
708 self.setTestResults(total=total, failed=failed, passed=passed,
709 warnings=warnings)
710
711 return rc
712