1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 import re
18 from twisted.python import log, failure
19 from twisted.spread import pb
20 from buildbot.process import buildstep
21 from buildbot.status.results import SUCCESS, WARNINGS, FAILURE
22 from buildbot.status.logfile import STDOUT, STDERR
23 from buildbot import config
24
25
26
27 from buildbot.process.properties import WithProperties
28 _hush_pyflakes = [WithProperties]
29 del _hush_pyflakes
30
32 """I run a single shell command on the buildslave. I return FAILURE if
33 the exit code of that command is non-zero, SUCCESS otherwise. To change
34 this behavior, override my .evaluateCommand method.
35
36 By default, a failure of this step will mark the whole build as FAILURE.
37 To override this, give me an argument of flunkOnFailure=False .
38
39 I create a single Log named 'log' which contains the output of the
40 command. To create additional summary Logs, override my .createSummary
41 method.
42
43 The shell command I run (a list of argv strings) can be provided in
44 several ways:
45 - a class-level .command attribute
46 - a command= parameter to my constructor (overrides .command)
47 - set explicitly with my .setCommand() method (overrides both)
48
49 @ivar command: a list of renderable objects (typically strings or
50 WithProperties instances). This will be used by start()
51 to create a RemoteShellCommand instance.
52
53 @ivar logfiles: a dict mapping log NAMEs to workdir-relative FILENAMEs
54 of their corresponding logfiles. The contents of the file
55 named FILENAME will be put into a LogFile named NAME, ina
56 something approximating real-time. (note that logfiles=
57 is actually handled by our parent class LoggingBuildStep)
58
59 @ivar lazylogfiles: Defaults to False. If True, logfiles will be tracked
60 `lazily', meaning they will only be added when and if
61 they are written to. Empty or nonexistent logfiles
62 will be omitted. (Also handled by class
63 LoggingBuildStep.)
64
65 """
66
67 name = "shell"
68 renderables = [ 'description', 'descriptionDone', 'slaveEnvironment', 'remote_kwargs', 'command', 'logfiles' ]
69 description = None
70 descriptionDone = None
71 command = None
72
73
74
75
76
77
78 flunkOnFailure = True
79
80 - def __init__(self, workdir=None,
81 description=None, descriptionDone=None,
82 command=None,
83 usePTY="slave-config",
84 **kwargs):
119
124
127
129 rkw = self.remote_kwargs
130 rkw['workdir'] = rkw['workdir'] or workdir
131
133 """
134 Get the current notion of the workdir. Note that this may change
135 between instantiation of the step and C{start}, as it is based on the
136 build's default workdir, and may even be C{None} before that point.
137 """
138 return self.remote_kwargs['workdir']
139
142
144 for x in commands:
145 if isinstance(x, (str, unicode)):
146 mainlist.append(x)
147 elif x != []:
148 self._flattenList(mainlist, x)
149
151 """Return a list of short strings to describe this step, for the
152 status display. This uses the first few words of the shell command.
153 You can replace this by setting .description in your subclass, or by
154 overriding this method to describe the step better.
155
156 @type done: boolean
157 @param done: whether the command is complete or not, to improve the
158 way the command is described. C{done=False} is used
159 while the command is still running, so a single
160 imperfect-tense verb is appropriate ('compiling',
161 'testing', ...) C{done=True} is used when the command
162 has finished, and the default getText() method adds some
163 text, so a simple noun is appropriate ('compile',
164 'tests' ...)
165 """
166
167 try:
168 if done and self.descriptionDone is not None:
169 return self.descriptionDone
170 if self.description is not None:
171 return self.description
172
173
174
175 if not self.command:
176 return ["???"]
177
178 words = self.command
179 if isinstance(words, (str, unicode)):
180 words = words.split()
181
182 try:
183 len(words)
184 except AttributeError:
185
186 return ["???"]
187
188 tmp = []
189 for x in words:
190 if isinstance(x, (str, unicode)):
191 tmp.append(x)
192 else:
193 self._flattenList(tmp, x)
194
195 if len(tmp) < 1:
196 return ["???"]
197 if len(tmp) == 1:
198 return ["'%s'" % tmp[0]]
199 if len(tmp) == 2:
200 return ["'%s" % tmp[0], "%s'" % tmp[1]]
201 return ["'%s" % tmp[0], "%s" % tmp[1], "...'"]
202
203 except:
204 log.err(failure.Failure(), "Error describing step")
205 return ["???"]
206
208
209
210
211
212
213
214 slaveEnv = self.slaveEnvironment
215 if slaveEnv:
216 if cmd.args['env'] is None:
217 cmd.args['env'] = {}
218 fullSlaveEnv = slaveEnv.copy()
219 fullSlaveEnv.update(cmd.args['env'])
220 cmd.args['env'] = fullSlaveEnv
221
222
223
247
261
262
263
265 name = "treesize"
266 command = ["du", "-s", "-k", "."]
267 description = "measuring tree size"
268 descriptionDone = "tree size measured"
269 kib = None
270
277
284
285 - def getText(self, cmd, results):
286 if self.kib is not None:
287 return ["treesize", "%d KiB" % self.kib]
288 return ["treesize", "unknown"]
289
290
292 name = "setproperty"
293 renderables = [ 'property' ]
294
295 - def __init__(self, property=None, extract_fn=None, strip=True, **kwargs):
311
313 if self.property:
314 if cmd.rc != 0:
315 return
316 result = cmd.logs['stdio'].getText()
317 if self.strip: result = result.strip()
318 propname = self.property
319 self.setProperty(propname, result, "SetProperty Step")
320 self.property_changes[propname] = result
321 else:
322 log = cmd.logs['stdio']
323 new_props = self.extract_fn(cmd.rc,
324 ''.join(log.getChunks([STDOUT], onlyText=True)),
325 ''.join(log.getChunks([STDERR], onlyText=True)))
326 for k,v in new_props.items():
327 self.setProperty(k, v, "SetProperty Step")
328 self.property_changes = new_props
329
331 if self.property_changes:
332 props_set = [ "%s: %r" % (k,v)
333 for k,v in self.property_changes.items() ]
334 self.addCompleteLog('property changes', "\n".join(props_set))
335
336 - def getText(self, cmd, results):
337 if len(self.property_changes) > 1:
338 return [ "%d properties set" % len(self.property_changes) ]
339 elif len(self.property_changes) == 1:
340 return [ "property '%s' set" % self.property_changes.keys()[0] ]
341 else:
342
343 return ShellCommand.getText(self, cmd, results)
344
353
355 """
356 FileWriter class that just puts received data into a buffer.
357
358 Used to upload a file from slave for inline processing rather than
359 writing into a file on master.
360 """
363
366
369
371 renderables = [ 'suppressionFile' ]
372
373 warnCount = 0
374 warningPattern = '.*warning[: ].*'
375
376 directoryEnterPattern = (u"make.*: Entering directory "
377 u"[\u2019\"`'](.*)[\u2019'`\"]")
378 directoryLeavePattern = "make.*: Leaving directory"
379 suppressionFile = None
380
381 commentEmptyLineRe = re.compile(r"^\s*(\#.*)?$")
382 suppressionLineRe = re.compile(r"^\s*(.+?)\s*:\s*(.+?)\s*(?:[:]\s*([0-9]+)(?:-([0-9]+))?\s*)?$")
383
384 - def __init__(self,
385 warningPattern=None, warningExtractor=None, maxWarnCount=None,
386 directoryEnterPattern=None, directoryLeavePattern=None,
387 suppressionFile=None, **kwargs):
416
418 """
419 This method can be used to add patters of warnings that should
420 not be counted.
421
422 It takes a single argument, a list of patterns.
423
424 Each pattern is a 4-tuple (FILE-RE, WARN-RE, START, END).
425
426 FILE-RE is a regular expression (string or compiled regexp), or None.
427 If None, the pattern matches all files, else only files matching the
428 regexp. If directoryEnterPattern is specified in the class constructor,
429 matching is against the full path name, eg. src/main.c.
430
431 WARN-RE is similarly a regular expression matched against the
432 text of the warning, or None to match all warnings.
433
434 START and END form an inclusive line number range to match against. If
435 START is None, there is no lower bound, similarly if END is none there
436 is no upper bound."""
437
438 for fileRe, warnRe, start, end in suppressionList:
439 if fileRe != None and isinstance(fileRe, basestring):
440 fileRe = re.compile(fileRe)
441 if warnRe != None and isinstance(warnRe, basestring):
442 warnRe = re.compile(warnRe)
443 self.suppressions.append((fileRe, warnRe, start, end))
444
446 """
447 Extract warning text as the whole line.
448 No file names or line numbers."""
449 return (None, None, line)
450
452 """
453 Extract file name, line number, and warning text as groups (1,2,3)
454 of warningPattern match."""
455 file = match.group(1)
456 lineNo = match.group(2)
457 if lineNo != None:
458 lineNo = int(lineNo)
459 text = match.group(3)
460 return (file, lineNo, text)
461
463 if self.suppressions:
464 (file, lineNo, text) = self.warningExtractor(self, line, match)
465 lineNo = lineNo and int(lineNo)
466
467 if file != None and file != "" and self.directoryStack:
468 currentDirectory = '/'.join(self.directoryStack)
469 if currentDirectory != None and currentDirectory != "":
470 file = "%s/%s" % (currentDirectory, file)
471
472
473 for fileRe, warnRe, start, end in self.suppressions:
474 if not (file == None or fileRe == None or fileRe.match(file)):
475 continue
476 if not (warnRe == None or warnRe.search(text)):
477 continue
478 if not ((start == None and end == None) or
479 (lineNo != None and start <= lineNo and end >= lineNo)):
480 continue
481 return
482
483 warnings.append(line)
484 self.warnCount += 1
485
503
525
527 """
528 Match log lines against warningPattern.
529
530 Warnings are collected into another log for this step, and the
531 build-wide 'warnings-count' is updated."""
532
533 self.warnCount = 0
534
535
536
537 wre = self.warningPattern
538 if isinstance(wre, str):
539 wre = re.compile(wre)
540
541 directoryEnterRe = self.directoryEnterPattern
542 if (directoryEnterRe != None
543 and isinstance(directoryEnterRe, basestring)):
544 directoryEnterRe = re.compile(directoryEnterRe)
545
546 directoryLeaveRe = self.directoryLeavePattern
547 if (directoryLeaveRe != None
548 and isinstance(directoryLeaveRe, basestring)):
549 directoryLeaveRe = re.compile(directoryLeaveRe)
550
551
552
553
554 warnings = []
555
556
557 for line in log.getText().split("\n"):
558 if directoryEnterRe:
559 match = directoryEnterRe.search(line)
560 if match:
561 self.directoryStack.append(match.group(1))
562 continue
563 if (directoryLeaveRe and
564 self.directoryStack and
565 directoryLeaveRe.search(line)):
566 self.directoryStack.pop()
567 continue
568
569 match = wre.match(line)
570 if match:
571 self.maybeAddWarning(warnings, line, match)
572
573
574
575 if self.warnCount:
576 self.addCompleteLog("warnings (%d)" % self.warnCount,
577 "\n".join(warnings) + "\n")
578
579 warnings_stat = self.step_status.getStatistic('warnings', 0)
580 self.step_status.setStatistic('warnings', warnings_stat + self.warnCount)
581
582 old_count = self.getProperty("warnings-count", 0)
583 self.setProperty("warnings-count", old_count + self.warnCount, "WarningCountingShellCommand")
584
585
593
594
595 -class Compile(WarningCountingShellCommand):
603
604 -class Test(WarningCountingShellCommand):
647
649 command=["prove", "--lib", "lib", "-r", "t"]
650 total = 0
651
653
654 lines = map(
655 lambda line : line.replace('\r\n','').replace('\r','').replace('\n',''),
656 self.getLog('stdio').readlines()
657 )
658
659 total = 0
660 passed = 0
661 failed = 0
662 rc = SUCCESS
663 if cmd.rc > 0:
664 rc = FAILURE
665
666
667 if "Test Summary Report" in lines:
668 test_summary_report_index = lines.index("Test Summary Report")
669 del lines[0:test_summary_report_index + 2]
670
671 re_test_result = re.compile("^Result: (PASS|FAIL)$|Tests: \d+ Failed: (\d+)\)|Files=\d+, Tests=(\d+)")
672
673 mos = map(lambda line: re_test_result.search(line), lines)
674 test_result_lines = [mo.groups() for mo in mos if mo]
675
676 for line in test_result_lines:
677 if line[0] == 'FAIL':
678 rc = FAILURE
679
680 if line[1]:
681 failed += int(line[1])
682 if line[2]:
683 total = int(line[2])
684
685 else:
686 re_test_result = re.compile("^(All tests successful)|(\d+)/(\d+) subtests failed|Files=\d+, Tests=(\d+),")
687
688 mos = map(lambda line: re_test_result.search(line), lines)
689 test_result_lines = [mo.groups() for mo in mos if mo]
690
691 if test_result_lines:
692 test_result_line = test_result_lines[0]
693
694 success = test_result_line[0]
695
696 if success:
697 failed = 0
698
699 test_totals_line = test_result_lines[1]
700 total_str = test_totals_line[3]
701 else:
702 failed_str = test_result_line[1]
703 failed = int(failed_str)
704
705 total_str = test_result_line[2]
706
707 rc = FAILURE
708
709 total = int(total_str)
710
711 warnings = 0
712 if self.warningPattern:
713 wre = self.warningPattern
714 if isinstance(wre, str):
715 wre = re.compile(wre)
716
717 warnings = len([l for l in lines if wre.search(l)])
718
719
720
721
722 if rc == SUCCESS and warnings:
723 rc = WARNINGS
724
725 if total:
726 passed = total - failed
727
728 self.setTestResults(total=total, failed=failed, passed=passed,
729 warnings=warnings)
730
731 return rc
732