1
2
3 import re
4 from twisted.python import log
5 from twisted.spread import pb
6 from buildbot.process.buildstep import LoggingBuildStep, RemoteShellCommand
7 from buildbot.process.buildstep import RemoteCommand
8 from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, STDOUT, STDERR
9 from buildbot.interfaces import BuildSlaveTooOldError
10
11
12
13 from buildbot.process.properties import WithProperties
14 _hush_pyflakes = [WithProperties]
15 del _hush_pyflakes
16
18 """I run a single shell command on the buildslave. I return FAILURE if
19 the exit code of that command is non-zero, SUCCESS otherwise. To change
20 this behavior, override my .evaluateCommand method.
21
22 By default, a failure of this step will mark the whole build as FAILURE.
23 To override this, give me an argument of flunkOnFailure=False .
24
25 I create a single Log named 'log' which contains the output of the
26 command. To create additional summary Logs, override my .createSummary
27 method.
28
29 The shell command I run (a list of argv strings) can be provided in
30 several ways:
31 - a class-level .command attribute
32 - a command= parameter to my constructor (overrides .command)
33 - set explicitly with my .setCommand() method (overrides both)
34
35 @ivar command: a list of renderable objects (typically strings or
36 WithProperties instances). This will be used by start()
37 to create a RemoteShellCommand instance.
38
39 @ivar logfiles: a dict mapping log NAMEs to workdir-relative FILENAMEs
40 of their corresponding logfiles. The contents of the file
41 named FILENAME will be put into a LogFile named NAME, ina
42 something approximating real-time. (note that logfiles=
43 is actually handled by our parent class LoggingBuildStep)
44
45 @ivar lazylogfiles: Defaults to False. If True, logfiles will be tracked
46 `lazily', meaning they will only be added when and if
47 they are written to. Empty or nonexistent logfiles
48 will be omitted. (Also handled by class
49 LoggingBuildStep.)
50
51 """
52
53 name = "shell"
54 description = None
55 descriptionDone = None
56 command = None
57
58
59
60
61
62
63 flunkOnFailure = True
64
65 - def __init__(self, workdir=None,
66 description=None, descriptionDone=None,
67 command=None,
68 usePTY="slave-config",
69 **kwargs):
104
107
109 rkw = self.remote_kwargs
110 rkw['workdir'] = rkw['workdir'] or workdir
111
114
116 """Return a list of short strings to describe this step, for the
117 status display. This uses the first few words of the shell command.
118 You can replace this by setting .description in your subclass, or by
119 overriding this method to describe the step better.
120
121 @type done: boolean
122 @param done: whether the command is complete or not, to improve the
123 way the command is described. C{done=False} is used
124 while the command is still running, so a single
125 imperfect-tense verb is appropriate ('compiling',
126 'testing', ...) C{done=True} is used when the command
127 has finished, and the default getText() method adds some
128 text, so a simple noun is appropriate ('compile',
129 'tests' ...)
130 """
131
132 try:
133 properties = self.build.getProperties()
134
135 if done and self.descriptionDone is not None:
136 return properties.render(self.descriptionDone)
137 if self.description is not None:
138 return properties.render(self.description)
139
140 words = self.command
141 if isinstance(words, (str, unicode)):
142 words = words.split()
143
144 words = properties.render(words)
145 if len(words) < 1:
146 return ["???"]
147 if len(words) == 1:
148 return ["'%s'" % words[0]]
149 if len(words) == 2:
150 return ["'%s" % words[0], "%s'" % words[1]]
151 return ["'%s" % words[0], "%s" % words[1], "...'"]
152 except:
153 log.msg("Error describing step")
154 log.err()
155 return ["???"]
156
172
173
174
176 if not self.logfiles:
177 return
178 if not self.slaveVersionIsOlderThan("shell", "2.1"):
179 return
180
181
182
183
184 msg1 = ("Warning: buildslave %s is too old "
185 "to understand logfiles=, ignoring it."
186 % self.getSlaveName())
187 msg2 = "You will have to pull this logfile (%s) manually."
188 log.msg(msg1)
189 for logname,remotefilevalue in self.logfiles.items():
190 remotefilename = remotefilevalue
191
192 if type(remotefilevalue) == dict:
193 remotefilename = remotefilevalue['filename']
194
195 newlog = self.addLog(logname)
196 newlog.addHeader(msg1 + "\n")
197 newlog.addHeader(msg2 % remotefilename + "\n")
198 newlog.finish()
199
200 self.logfiles = {}
201
226
227
228
230 name = "treesize"
231 command = ["du", "-s", "-k", "."]
232 description = "measuring tree size"
233 descriptionDone = "tree size measured"
234 kib = None
235
238
245
252
253 - def getText(self, cmd, results):
254 if self.kib is not None:
255 return ["treesize", "%d KiB" % self.kib]
256 return ["treesize", "unknown"]
257
259 name = "setproperty"
260
261 - def __init__(self, property=None, extract_fn=None, strip=True, **kwargs):
276
292
294 props_set = [ "%s: %r" % (k,v) for k,v in self.property_changes.items() ]
295 self.addCompleteLog('property changes', "\n".join(props_set))
296
297 - def getText(self, cmd, results):
298 if self.property_changes:
299 return [ "set props:" ] + self.property_changes.keys()
300 else:
301 return [ "no change" ]
302
311
313 """
314 FileWriter class that just puts received data into a buffer.
315
316 Used to upload a file from slave for inline processing rather than
317 writing into a file on master.
318 """
321
324
327
329 """
330 Remote command subclass used to run an internal file upload command on the
331 slave. We do not need any progress updates from such command, so override
332 remoteUpdate() with an empty method.
333 """
336
338 warnCount = 0
339 warningPattern = '.*warning[: ].*'
340
341 directoryEnterPattern = "make.*: Entering directory [\"`'](.*)['`\"]"
342 directoryLeavePattern = "make.*: Leaving directory"
343 suppressionFile = None
344
345 commentEmptyLineRe = re.compile(r"^\s*(\#.*)?$")
346 suppressionLineRe = re.compile(r"^\s*(.+?)\s*:\s*(.+?)\s*(?:[:]\s*([0-9]+)(?:-([0-9]+))?\s*)?$")
347
348 - def __init__(self, workdir=None,
349 warningPattern=None, warningExtractor=None,
350 directoryEnterPattern=None, directoryLeavePattern=None,
351 suppressionFile=None, **kwargs):
379
384
386 """
387 This method can be used to add patters of warnings that should
388 not be counted.
389
390 It takes a single argument, a list of patterns.
391
392 Each pattern is a 4-tuple (FILE-RE, WARN-RE, START, END).
393
394 FILE-RE is a regular expression (string or compiled regexp), or None.
395 If None, the pattern matches all files, else only files matching the
396 regexp. If directoryEnterPattern is specified in the class constructor,
397 matching is against the full path name, eg. src/main.c.
398
399 WARN-RE is similarly a regular expression matched against the
400 text of the warning, or None to match all warnings.
401
402 START and END form an inclusive line number range to match against. If
403 START is None, there is no lower bound, similarly if END is none there
404 is no upper bound."""
405
406 for fileRe, warnRe, start, end in suppressionList:
407 if fileRe != None and isinstance(fileRe, str):
408 fileRe = re.compile(fileRe)
409 if warnRe != None and isinstance(warnRe, str):
410 warnRe = re.compile(warnRe)
411 self.suppressions.append((fileRe, warnRe, start, end))
412
414 """
415 Extract warning text as the whole line.
416 No file names or line numbers."""
417 return (None, None, line)
418
420 """
421 Extract file name, line number, and warning text as groups (1,2,3)
422 of warningPattern match."""
423 file = match.group(1)
424 lineNo = match.group(2)
425 if lineNo != None:
426 lineNo = int(lineNo)
427 text = match.group(3)
428 return (file, lineNo, text)
429
431 if self.suppressions:
432 (file, lineNo, text) = self.warningExtractor(self, line, match)
433
434 if file != None and file != "" and self.directoryStack:
435 currentDirectory = self.directoryStack[-1]
436 if currentDirectory != None and currentDirectory != "":
437 file = "%s/%s" % (currentDirectory, file)
438
439
440 for fileRe, warnRe, start, end in self.suppressions:
441 if ( (file == None or fileRe == None or fileRe.search(file)) and
442 (warnRe == None or warnRe.search(text)) and
443 ((start == None and end == None) or
444 (lineNo != None and start <= lineNo and end >= lineNo)) ):
445 return
446
447 warnings.append(line)
448 self.warnCount += 1
449
474
496
498 """
499 Match log lines against warningPattern.
500
501 Warnings are collected into another log for this step, and the
502 build-wide 'warnings-count' is updated."""
503
504 self.warnCount = 0
505
506
507
508 if not self.warningPattern:
509 return
510
511 wre = self.warningPattern
512 if isinstance(wre, str):
513 wre = re.compile(wre)
514
515 directoryEnterRe = self.directoryEnterPattern
516 if directoryEnterRe != None and isinstance(directoryEnterRe, str):
517 directoryEnterRe = re.compile(directoryEnterRe)
518
519 directoryLeaveRe = self.directoryLeavePattern
520 if directoryLeaveRe != None and isinstance(directoryLeaveRe, str):
521 directoryLeaveRe = re.compile(directoryLeaveRe)
522
523
524
525
526 warnings = []
527
528
529 for line in log.getText().split("\n"):
530 if directoryEnterRe:
531 match = directoryEnterRe.search(line)
532 if match:
533 self.directoryStack.append(match.group(1))
534 if (directoryLeaveRe and
535 self.directoryStack and
536 directoryLeaveRe.search(line)):
537 self.directoryStack.pop()
538
539 match = wre.match(line)
540 if match:
541 self.maybeAddWarning(warnings, line, match)
542
543
544
545 if self.warnCount:
546 self.addCompleteLog("warnings (%d)" % self.warnCount,
547 "\n".join(warnings) + "\n")
548
549 warnings_stat = self.step_status.getStatistic('warnings', 0)
550 self.step_status.setStatistic('warnings', warnings_stat + self.warnCount)
551
552 try:
553 old_count = self.getProperty("warnings-count")
554 except KeyError:
555 old_count = 0
556 self.setProperty("warnings-count", old_count + self.warnCount, "WarningCountingShellCommand")
557
558
565
566
567 -class Compile(WarningCountingShellCommand):
584
585 -class Test(WarningCountingShellCommand):
627
629 command=["prove", "--lib", "lib", "-r", "t"]
630 total = 0
631
633
634 lines = map(
635 lambda line : line.replace('\r\n','').replace('\r','').replace('\n',''),
636 self.getLog('stdio').readlines()
637 )
638
639 total = 0
640 passed = 0
641 failed = 0
642 rc = SUCCESS
643 if cmd.rc > 0:
644 rc = FAILURE
645
646
647 if "Test Summary Report" in lines:
648 test_summary_report_index = lines.index("Test Summary Report")
649 del lines[0:test_summary_report_index + 2]
650
651 re_test_result = re.compile("^Result: (PASS|FAIL)$|Tests: \d+ Failed: (\d+)\)|Files=\d+, Tests=(\d+)")
652
653 mos = map(lambda line: re_test_result.search(line), lines)
654 test_result_lines = [mo.groups() for mo in mos if mo]
655
656 for line in test_result_lines:
657 if line[0] == 'FAIL':
658 rc = FAILURE
659
660 if line[1]:
661 failed += int(line[1])
662 if line[2]:
663 total = int(line[2])
664
665 else:
666 re_test_result = re.compile("^(All tests successful)|(\d+)/(\d+) subtests failed|Files=\d+, Tests=(\d+),")
667
668 mos = map(lambda line: re_test_result.search(line), lines)
669 test_result_lines = [mo.groups() for mo in mos if mo]
670
671 if test_result_lines:
672 test_result_line = test_result_lines[0]
673
674 success = test_result_line[0]
675
676 if success:
677 failed = 0
678
679 test_totals_line = test_result_lines[1]
680 total_str = test_totals_line[3]
681 else:
682 failed_str = test_result_line[1]
683 failed = int(failed_str)
684
685 total_str = test_result_line[2]
686
687 rc = FAILURE
688
689 total = int(total_str)
690
691 warnings = 0
692 if self.warningPattern:
693 wre = self.warningPattern
694 if isinstance(wre, str):
695 wre = re.compile(wre)
696
697 warnings = len([l for l in lines if wre.search(l)])
698
699
700
701
702 if rc == SUCCESS and warnings:
703 rc = WARNINGS
704
705 if total:
706 passed = total - failed
707
708 self.setTestResults(total=total, failed=failed, passed=passed,
709 warnings=warnings)
710
711 return rc
712