Source Code for Module buildbot.changes.p4poller

  1  # -*- test-case-name: buildbot.test.test_p4poller -*- 
  2   
  3  # Many thanks to Dave Peticolas for contributing this module 
  4   
  5  import re 
  6  import time 
  7  import os 
  8   
  9  from twisted.python import log 
 10  from twisted.internet import defer, reactor 
 11  from twisted.internet.utils import getProcessOutput 
 12  from twisted.internet.task import LoopingCall 
 13   
 14  from buildbot import util 
 15  from buildbot.changes import base, changes 
 16   
 17 -class P4PollerError(Exception): 
 18      """Something went wrong with the poll. This is used as a distinctive 
 19      exception type so that unit tests can detect and ignore it.""" 
 20   
 21 -def get_simple_split(branchfile): 
 22      """Splits the branchfile argument and assuming branch is  
 23         the first path component in branchfile, will return 
 24         branch and file else None.""" 
 25   
 26      index = branchfile.find('/') 
 27      if index == -1: return None, None 
 28      branch, file = branchfile.split('/', 1) 
 29      return branch, file 
 30   
 31 -class P4Source(base.ChangeSource, util.ComparableMixin): 
 32      """This source will poll a perforce repository for changes and submit 
 33      them to the change master.""" 
 34   
 35      compare_attrs = ["p4port", "p4user", "p4passwd", "p4base", 
 36                       "p4bin", "pollinterval"] 
 37   
 38      env_vars = ["P4CLIENT", "P4PORT", "P4PASSWD", "P4USER", 
 39                  "P4CHARSET"] 
 40   
 41      changes_line_re = re.compile( 
 42              r"Change (?P<num>\d+) on \S+ by \S+@\S+ '.*'$") 
 43      describe_header_re = re.compile( 
 44              r"Change \d+ by (?P<who>\S+)@\S+ on (?P<when>.+)$") 
 45      file_re = re.compile(r"^\.\.\. (?P<path>[^#]+)#\d+ [/\w]+$") 
 46      datefmt = '%Y/%m/%d %H:%M:%S' 
 47   
 48      parent = None # filled in when we're added 
 49      last_change = None 
 50      loop = None 
 51      working = False 
 52   
 53 -    def __init__(self, p4port=None, p4user=None, p4passwd=None, 
 54                   p4base='//', p4bin='p4', 
 55                   split_file=lambda branchfile: (None, branchfile), 
 56                   pollinterval=60 * 10, histmax=None): 
 57          """ 
 58          @type  p4port:       string 
 59          @param p4port:       p4 port definition (host:portno) 
 60          @type  p4user:       string 
 61          @param p4user:       p4 user 
 62          @type  p4passwd:     string 
 63          @param p4passwd:     p4 passwd 
 64          @type  p4base:       string 
 65          @param p4base:       p4 file specification to limit a poll to 
 66                               without the trailing '...' (i.e., //) 
 67          @type  p4bin:        string 
 68          @param p4bin:        path to p4 binary, defaults to just 'p4' 
 69          @type  split_file:   func 
 70          $param split_file:   splits a filename into branch and filename. 
 71          @type  pollinterval: int 
 72          @param pollinterval: interval in seconds between polls 
 73          @type  histmax:      int 
 74          @param histmax:      (obsolete) maximum number of changes to look back through. 
 75                               ignored; accepted for backwards compatibility. 
 76          """ 
 77   
 78          self.p4port = p4port 
 79          self.p4user = p4user 
 80          self.p4passwd = p4passwd 
 81          self.p4base = p4base 
 82          self.p4bin = p4bin 
 83          self.split_file = split_file 
 84          self.pollinterval = pollinterval 
 85          self.loop = LoopingCall(self.checkp4) 
 86   
 87 -    def startService(self): 
 88          base.ChangeSource.startService(self) 
 89   
 90          # Don't start the loop just yet because the reactor isn't running. 
 91          # Give it a chance to go and install our SIGCHLD handler before 
 92          # spawning processes. 
 93          reactor.callLater(0, self.loop.start, self.pollinterval) 
 94   
 95 -    def stopService(self): 
 96          self.loop.stop() 
 97          return base.ChangeSource.stopService(self) 
 98   
 99 -    def describe(self): 
100          return "p4source %s %s" % (self.p4port, self.p4base) 
101   
102 -    def checkp4(self): 
103          # Our return value is only used for unit testing. 
104          if self.working: 
105              log.msg("Skipping checkp4 because last one has not finished") 
106              return defer.succeed(None) 
107          else: 
108              self.working = True 
109              d = self._get_changes() 
110              d.addCallback(self._process_changes) 
111              d.addCallbacks(self._finished_ok, self._finished_failure) 
112              return d 
113   
114 -    def _finished_ok(self, res): 
115          assert self.working 
116          self.working = False 
117          return res 
118   
119 -    def _finished_failure(self, res): 
120          assert self.working 
121          self.working = False 
122   
123          # Again, the return value is only for unit testing. If there's a 
124          # failure, log it so it isn't lost. Use log.err to make sure unit 
125          # tests flunk if there was a problem. 
126          log.err(res, 'P4 poll failed') 
127          return None 
128   
129 -    def _get_process_output(self, args): 
130          env = dict([(e, os.environ.get(e)) for e in self.env_vars if os.environ.get(e)]) 
131          d = getProcessOutput(self.p4bin, args, env) 
132          return d 
133   
134 -    def _get_changes(self): 
135          args = [] 
136          if self.p4port: 
137              args.extend(['-p', self.p4port]) 
138          if self.p4user: 
139              args.extend(['-u', self.p4user]) 
140          if self.p4passwd: 
141              args.extend(['-P', self.p4passwd]) 
142          args.extend(['changes']) 
143          if self.last_change is not None: 
144              args.extend(['%s...@%d,now' % (self.p4base, self.last_change+1)]) 
145          else: 
146              args.extend(['-m', '1', '%s...' % (self.p4base,)]) 
147          return self._get_process_output(args) 
148   
149 -    def _process_changes(self, result): 
150          last_change = self.last_change 
151          changelists = [] 
152          for line in result.split('\n'): 
153              line = line.strip() 
154              if not line: continue 
155              m = self.changes_line_re.match(line) 
156              if not m: 
157                  raise P4PollerError("Unexpected 'p4 changes' output: %r" % result) 
158              num = int(m.group('num')) 
159              if last_change is None: 
160                  log.msg('P4Poller: starting at change %d' % num) 
161                  self.last_change = num 
162                  return [] 
163              changelists.append(num) 
164          changelists.reverse() # oldest first 
165   
166          # Retrieve each sequentially. 
167          d = defer.succeed(None) 
168          for c in changelists: 
169              d.addCallback(self._get_describe, c) 
170              d.addCallback(self._process_describe, c) 
171          return d 
172   
173 -    def _get_describe(self, dummy, num): 
174          args = [] 
175          if self.p4port: 
176              args.extend(['-p', self.p4port]) 
177          if self.p4user: 
178              args.extend(['-u', self.p4user]) 
179          if self.p4passwd: 
180              args.extend(['-P', self.p4passwd]) 
181          args.extend(['describe', '-s', str(num)]) 
182          return self._get_process_output(args) 
183   
184 -    def _process_describe(self, result, num): 
185          lines = result.split('\n') 
186          # SF#1555985: Wade Brainerd reports a stray ^M at the end of the date 
187          # field. The rstrip() is intended to remove that. 
188          lines[0] = lines[0].rstrip() 
189          m = self.describe_header_re.match(lines[0]) 
190          if not m: 
191              raise P4PollerError("Unexpected 'p4 describe -s' result: %r" % result) 
192          who = m.group('who') 
193          when = time.mktime(time.strptime(m.group('when'), self.datefmt)) 
194          comments = '' 
195          while not lines[0].startswith('Affected files'): 
196              comments += lines.pop(0) + '\n' 
197          lines.pop(0) # affected files 
198   
199          branch_files = {} # dict for branch mapped to file(s) 
200          while lines: 
201              line = lines.pop(0).strip() 
202              if not line: continue 
203              m = self.file_re.match(line) 
204              if not m: 
205                  raise P4PollerError("Invalid file line: %r" % line) 
206              path = m.group('path') 
207              if path.startswith(self.p4base): 
208                  branch, file = self.split_file(path[len(self.p4base):]) 
209                  if (branch == None and file == None): continue 
210                  if branch_files.has_key(branch): 
211                      branch_files[branch].append(file) 
212                  else: 
213                      branch_files[branch] = [file] 
214   
215          for branch in branch_files: 
216              c = changes.Change(who=who, 
217                                 files=branch_files[branch], 
218                                 comments=comments, 
219                                 revision=str(num), 
220                                 when=when, 
221                                 branch=branch) 
222              self.parent.addChange(c) 
223   
224          self.last_change = num 
225