1   
  2   
  3  import re 
  4  from twisted.python import log 
  5  from twisted.spread import pb 
  6  from buildbot.process.buildstep import LoggingBuildStep, RemoteShellCommand 
  7  from buildbot.process.buildstep import RemoteCommand 
  8  from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, STDOUT, STDERR 
  9   
 10   
 11   
 12  from buildbot.process.properties import WithProperties 
 13  _hush_pyflakes = [WithProperties] 
 14  del _hush_pyflakes 
 15   
 17      """I run a single shell command on the buildslave. I return FAILURE if 
 18      the exit code of that command is non-zero, SUCCESS otherwise. To change 
 19      this behavior, override my .evaluateCommand method. 
 20   
 21      By default, a failure of this step will mark the whole build as FAILURE. 
 22      To override this, give me an argument of flunkOnFailure=False . 
 23   
 24      I create a single Log named 'log' which contains the output of the 
 25      command. To create additional summary Logs, override my .createSummary 
 26      method. 
 27   
 28      The shell command I run (a list of argv strings) can be provided in 
 29      several ways: 
 30        - a class-level .command attribute 
 31        - a command= parameter to my constructor (overrides .command) 
 32        - set explicitly with my .setCommand() method (overrides both) 
 33   
 34      @ivar command: a list of renderable objects (typically strings or 
 35                     WithProperties instances). This will be used by start() 
 36                     to create a RemoteShellCommand instance. 
 37   
 38      @ivar logfiles: a dict mapping log NAMEs to workdir-relative FILENAMEs 
 39                      of their corresponding logfiles. The contents of the file 
 40                      named FILENAME will be put into a LogFile named NAME, ina 
 41                      something approximating real-time. (note that logfiles= 
 42                      is actually handled by our parent class LoggingBuildStep) 
 43   
 44      @ivar lazylogfiles: Defaults to False. If True, logfiles will be tracked 
 45                          `lazily', meaning they will only be added when and if 
 46                          they are written to. Empty or nonexistent logfiles 
 47                          will be omitted. (Also handled by class 
 48                          LoggingBuildStep.) 
 49   
 50      """ 
 51   
 52      name = "shell" 
 53      description = None  
 54      descriptionDone = None  
 55      command = None  
 56       
 57       
 58       
 59   
 60       
 61       
 62      flunkOnFailure = True 
 63   
 64 -    def __init__(self, workdir=None, 
 65                   description=None, descriptionDone=None, 
 66                   command=None, 
 67                   usePTY="slave-config", 
 68                   **kwargs): 
 103   
105          rkw = self.remote_kwargs 
106          rkw['workdir'] = rkw['workdir'] or workdir 
 107   
110   
112          """Return a list of short strings to describe this step, for the 
113          status display. This uses the first few words of the shell command. 
114          You can replace this by setting .description in your subclass, or by 
115          overriding this method to describe the step better. 
116   
117          @type  done: boolean 
118          @param done: whether the command is complete or not, to improve the 
119                       way the command is described. C{done=False} is used 
120                       while the command is still running, so a single 
121                       imperfect-tense verb is appropriate ('compiling', 
122                       'testing', ...) C{done=True} is used when the command 
123                       has finished, and the default getText() method adds some 
124                       text, so a simple noun is appropriate ('compile', 
125                       'tests' ...) 
126          """ 
127   
128          if done and self.descriptionDone is not None: 
129              return list(self.descriptionDone) 
130          if self.description is not None: 
131              return list(self.description) 
132   
133          properties = self.build.getProperties() 
134          words = self.command 
135          if isinstance(words, (str, unicode)): 
136              words = words.split() 
137           
138          words = properties.render(words) 
139          if len(words) < 1: 
140              return ["???"] 
141          if len(words) == 1: 
142              return ["'%s'" % words[0]] 
143          if len(words) == 2: 
144              return ["'%s" % words[0], "%s'" % words[1]] 
145          return ["'%s" % words[0], "%s" % words[1], "...'"] 
 146   
148           
149           
150           
151           
152           
153           
154          properties = self.build.getProperties() 
155          slaveEnv = self.build.slaveEnvironment 
156          if slaveEnv: 
157              if cmd.args['env'] is None: 
158                  cmd.args['env'] = {} 
159              fullSlaveEnv = slaveEnv.copy() 
160              fullSlaveEnv.update(cmd.args['env']) 
161              cmd.args['env'] = properties.render(fullSlaveEnv) 
 162               
163               
164   
166          if not self.logfiles: 
167              return  
168          if not self.slaveVersionIsOlderThan("shell", "2.1"): 
169              return  
170           
171           
172           
173           
174          msg1 = ("Warning: buildslave %s is too old " 
175                  "to understand logfiles=, ignoring it." 
176                 % self.getSlaveName()) 
177          msg2 = "You will have to pull this logfile (%s) manually." 
178          log.msg(msg1) 
179          for logname,remotefilevalue in self.logfiles.items(): 
180              remotefilename = remotefilevalue 
181               
182              if type(remotefilevalue) == dict: 
183                  remotefilename = remotefilevalue['filename'] 
184   
185              newlog = self.addLog(logname) 
186              newlog.addHeader(msg1 + "\n") 
187              newlog.addHeader(msg2 % remotefilename + "\n") 
188              newlog.finish() 
189           
190          self.logfiles = {} 
 191   
 216   
217   
218   
220      name = "treesize" 
221      command = ["du", "-s", "-k", "."] 
222      kib = None 
223   
225          out = cmd.logs['stdio'].getText() 
226          m = re.search(r'^(\d+)', out) 
227          if m: 
228              self.kib = int(m.group(1)) 
229              self.setProperty("tree-size-KiB", self.kib, "treesize") 
 230   
237   
238 -    def getText(self, cmd, results): 
 239          if self.kib is not None: 
240              return ["treesize", "%d KiB" % self.kib] 
241          return ["treesize", "unknown"] 
  242   
244      name = "setproperty" 
245   
247          self.property = None 
248          self.extract_fn = None 
249          self.strip = True 
250   
251          if kwargs.has_key('property'): 
252              self.property = kwargs['property'] 
253              del kwargs['property'] 
254          if kwargs.has_key('extract_fn'): 
255              self.extract_fn = kwargs['extract_fn'] 
256              del kwargs['extract_fn'] 
257          if kwargs.has_key('strip'): 
258              self.strip = kwargs['strip'] 
259              del kwargs['strip'] 
260   
261          ShellCommand.__init__(self, **kwargs) 
262   
263          self.addFactoryArguments(property=self.property) 
264          self.addFactoryArguments(extract_fn=self.extract_fn) 
265          self.addFactoryArguments(strip=self.strip) 
266   
267          assert self.property or self.extract_fn, \ 
268              "SetProperty step needs either property= or extract_fn=" 
269   
270          self.property_changes = {} 
 271   
273          if self.property: 
274              result = cmd.logs['stdio'].getText() 
275              if self.strip: result = result.strip() 
276              propname = self.build.getProperties().render(self.property) 
277              self.setProperty(propname, result, "SetProperty Step") 
278              self.property_changes[propname] = result 
279          else: 
280              log = cmd.logs['stdio'] 
281              new_props = self.extract_fn(cmd.rc, 
282                      ''.join(log.getChunks([STDOUT], onlyText=True)), 
283                      ''.join(log.getChunks([STDERR], onlyText=True))) 
284              for k,v in new_props.items(): 
285                  self.setProperty(k, v, "SetProperty Step") 
286              self.property_changes = new_props 
 287   
289          props_set = [ "%s: %r" % (k,v) for k,v in self.property_changes.items() ] 
290          self.addCompleteLog('property changes', "\n".join(props_set)) 
 291   
292 -    def getText(self, cmd, results): 
 293          if self.property_changes: 
294              return [ "set props:" ] + self.property_changes.keys() 
295          else: 
296              return [ "no change" ] 
  297   
306   
308      """ 
309      FileWriter class that just puts received data into a buffer. 
310   
311      Used to upload a file from slave for inline processing rather than 
312      writing into a file on master. 
313      """ 
316   
319   
 322   
324      """ 
325      Remote command subclass used to run an internal file upload command on the 
326      slave. We do not need any progress updates from such command, so override 
327      remoteUpdate() with an empty method. 
328      """ 
 331   
333      warnCount = 0 
334      warningPattern = '.*warning[: ].*' 
335       
336      directoryEnterPattern = "make.*: Entering directory [\"`'](.*)['`\"]" 
337      directoryLeavePattern = "make.*: Leaving directory" 
338      suppressionFile = None 
339   
340      commentEmptyLineRe = re.compile(r"^\s*(\#.*)?$") 
341      suppressionLineRe = re.compile(r"^\s*(.+?)\s*:\s*(.+?)\s*(?:[:]\s*([0-9]+)(?:-([0-9]+))?\s*)?$") 
342   
343 -    def __init__(self, workdir=None, 
344                   warningPattern=None, warningExtractor=None, 
345                   directoryEnterPattern=None, directoryLeavePattern=None, 
346                   suppressionFile=None, **kwargs): 
 374   
379   
381          """ 
382          This method can be used to add patters of warnings that should 
383          not be counted. 
384   
385          It takes a single argument, a list of patterns. 
386   
387          Each pattern is a 4-tuple (FILE-RE, WARN-RE, START, END). 
388   
389          FILE-RE is a regular expression (string or compiled regexp), or None. 
390          If None, the pattern matches all files, else only files matching the 
391          regexp. If directoryEnterPattern is specified in the class constructor, 
392          matching is against the full path name, eg. src/main.c. 
393   
394          WARN-RE is similarly a regular expression matched against the 
395          text of the warning, or None to match all warnings. 
396   
397          START and END form an inclusive line number range to match against. If 
398          START is None, there is no lower bound, similarly if END is none there 
399          is no upper bound.""" 
400   
401          for fileRe, warnRe, start, end in suppressionList: 
402              if fileRe != None and isinstance(fileRe, str): 
403                  fileRe = re.compile(fileRe) 
404              if warnRe != None and isinstance(warnRe, str): 
405                  warnRe = re.compile(warnRe) 
406              self.suppressions.append((fileRe, warnRe, start, end)) 
 407   
409          """ 
410          Extract warning text as the whole line. 
411          No file names or line numbers.""" 
412          return (None, None, line) 
 413   
415          """ 
416          Extract file name, line number, and warning text as groups (1,2,3) 
417          of warningPattern match.""" 
418          file = match.group(1) 
419          lineNo = match.group(2) 
420          if lineNo != None: 
421              lineNo = int(lineNo) 
422          text = match.group(3) 
423          return (file, lineNo, text) 
 424   
426          if self.suppressions: 
427              (file, lineNo, text) = self.warningExtractor(self, line, match) 
428   
429              if file != None and file != "" and self.directoryStack: 
430                  currentDirectory = self.directoryStack[-1] 
431                  if currentDirectory != None and currentDirectory != "": 
432                      file = "%s/%s" % (currentDirectory, file) 
433   
434               
435              for fileRe, warnRe, start, end in self.suppressions: 
436                  if ( (file == None or fileRe == None or fileRe.search(file)) and 
437                       (warnRe == None or  warnRe.search(text)) and 
438                       lineNo != None and 
439                       (start == None or start <= lineNo) and 
440                       (end == None or end >= lineNo) ): 
441                      return 
442   
443          warnings.append(line) 
444          self.warnCount += 1 
 445   
470   
492   
494          self.warnCount = 0 
495   
496           
497           
498          if not self.warningPattern: 
499              return 
500   
501          wre = self.warningPattern 
502          if isinstance(wre, str): 
503              wre = re.compile(wre) 
504   
505          directoryEnterRe = self.directoryEnterPattern 
506          if directoryEnterRe != None and isinstance(directoryEnterRe, str): 
507              directoryEnterRe = re.compile(directoryEnterRe) 
508   
509          directoryLeaveRe = self.directoryLeavePattern 
510          if directoryLeaveRe != None and isinstance(directoryLeaveRe, str): 
511              directoryLeaveRe = re.compile(directoryLeaveRe) 
512   
513           
514           
515           
516          warnings = [] 
517           
518           
519          for line in log.getText().split("\n"): 
520              if directoryEnterRe: 
521                  match = directoryEnterRe.search(line) 
522                  if match: 
523                      self.directoryStack.append(match.group(1)) 
524                  if (directoryLeaveRe and 
525                      self.directoryStack and 
526                      directoryLeaveRe.search(line)): 
527                          self.directoryStack.pop() 
528   
529              match = wre.match(line) 
530              if match: 
531                  self.maybeAddWarning(warnings, line, match) 
532   
533           
534           
535          if self.warnCount: 
536              self.addCompleteLog("warnings", "\n".join(warnings) + "\n") 
537   
538          warnings_stat = self.step_status.getStatistic('warnings', 0) 
539          self.step_status.setStatistic('warnings', warnings_stat + self.warnCount) 
540   
541          try: 
542              old_count = self.getProperty("warnings-count") 
543          except KeyError: 
544              old_count = 0 
545          self.setProperty("warnings-count", old_count + self.warnCount, "WarningCountingShellCommand") 
 546   
547   
 554   
555   
556 -class Compile(WarningCountingShellCommand): 
 573   
574 -class Test(WarningCountingShellCommand): 
 616   
618      command=["prove", "--lib", "lib", "-r", "t"] 
619      total = 0 
620   
622           
623          lines = map( 
624              lambda line : line.replace('\r\n','').replace('\r','').replace('\n',''), 
625              self.getLog('stdio').readlines() 
626              ) 
627   
628          total = 0 
629          passed = 0 
630          failed = 0 
631          rc = cmd.rc 
632   
633           
634          try: 
635              test_summary_report_index = lines.index("Test Summary Report") 
636   
637              del lines[0:test_summary_report_index + 2] 
638   
639              re_test_result = re.compile("^Result: (PASS|FAIL)$|Tests: \d+ Failed: (\d+)\)|Files=\d+, Tests=(\d+)") 
640   
641              mos = map(lambda line: re_test_result.search(line), lines) 
642              test_result_lines = [mo.groups() for mo in mos if mo] 
643   
644              for line in test_result_lines: 
645                  if line[0] == 'PASS': 
646                      rc = SUCCESS 
647                  elif line[0] == 'FAIL': 
648                      rc = FAILURE 
649                  elif line[1]: 
650                      failed += int(line[1]) 
651                  elif line[2]: 
652                      total = int(line[2]) 
653   
654          except ValueError:  
655              re_test_result = re.compile("^(All tests successful)|(\d+)/(\d+) subtests failed|Files=\d+, Tests=(\d+),") 
656   
657              mos = map(lambda line: re_test_result.search(line), lines) 
658              test_result_lines = [mo.groups() for mo in mos if mo] 
659   
660              if test_result_lines: 
661                  test_result_line = test_result_lines[0] 
662   
663                  success = test_result_line[0] 
664   
665                  if success: 
666                      failed = 0 
667   
668                      test_totals_line = test_result_lines[1] 
669                      total_str = test_totals_line[3] 
670                      rc = SUCCESS 
671                  else: 
672                      failed_str = test_result_line[1] 
673                      failed = int(failed_str) 
674   
675                      total_str = test_result_line[2] 
676   
677                      rc = FAILURE 
678   
679                  total = int(total_str) 
680   
681          warnings = 0 
682          if self.warningPattern: 
683              wre = self.warningPattern 
684              if isinstance(wre, str): 
685                  wre = re.compile(wre) 
686   
687              warnings = len([l for l in lines if wre.search(l)]) 
688   
689               
690               
691               
692              if rc == SUCCESS and warnings: 
693                  rc = WARNINGS 
694   
695          if total: 
696              passed = total - failed 
697   
698              self.setTestResults(total=total, failed=failed, passed=passed, 
699                                  warnings=warnings) 
700   
701          return rc 
  702