Source Code for Module buildbot.changes.p4poller

  1  # This file is part of Buildbot.  Buildbot is free software: you can 
  2  # redistribute it and/or modify it under the terms of the GNU General Public 
  3  # License as published by the Free Software Foundation, version 2. 
  4  # 
  5  # This program is distributed in the hope that it will be useful, but WITHOUT 
  6  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
  7  # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more 
  8  # details. 
  9  # 
 10  # You should have received a copy of the GNU General Public License along with 
 11  # this program; if not, write to the Free Software Foundation, Inc., 51 
 12  # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 
 13  # 
 14  # Copyright Buildbot Team Members 
 15   
 16   
 17  # Many thanks to Dave Peticolas for contributing this module 
 18   
 19  import re 
 20  import time 
 21  import os 
 22   
 23  from twisted.python import log 
 24  from twisted.internet import defer, reactor 
 25  from twisted.internet.utils import getProcessOutput 
 26  from twisted.internet.task import LoopingCall 
 27   
 28  from buildbot import util 
 29  from buildbot.changes import base, changes 
 30   
 31 -class P4PollerError(Exception): 
 32      """Something went wrong with the poll. This is used as a distinctive 
 33      exception type so that unit tests can detect and ignore it.""" 
 34   
 35 -def get_simple_split(branchfile): 
 36      """Splits the branchfile argument and assuming branch is  
 37         the first path component in branchfile, will return 
 38         branch and file else None.""" 
 39   
 40      index = branchfile.find('/') 
 41      if index == -1: return None, None 
 42      branch, file = branchfile.split('/', 1) 
 43      return branch, file 
 44   
 45 -class P4Source(base.ChangeSource, util.ComparableMixin): 
 46      """This source will poll a perforce repository for changes and submit 
 47      them to the change master.""" 
 48   
 49      compare_attrs = ["p4port", "p4user", "p4passwd", "p4base", 
 50                       "p4bin", "pollinterval"] 
 51   
 52      env_vars = ["P4CLIENT", "P4PORT", "P4PASSWD", "P4USER", 
 53                  "P4CHARSET"] 
 54   
 55      changes_line_re = re.compile( 
 56              r"Change (?P<num>\d+) on \S+ by \S+@\S+ '.*'$") 
 57      describe_header_re = re.compile( 
 58              r"Change \d+ by (?P<who>\S+)@\S+ on (?P<when>.+)$") 
 59      file_re = re.compile(r"^\.\.\. (?P<path>[^#]+)#\d+ [/\w]+$") 
 60      datefmt = '%Y/%m/%d %H:%M:%S' 
 61   
 62      parent = None # filled in when we're added 
 63      last_change = None 
 64      loop = None 
 65      working = False 
 66   
 67 -    def __init__(self, p4port=None, p4user=None, p4passwd=None, 
 68                   p4base='//', p4bin='p4', 
 69                   split_file=lambda branchfile: (None, branchfile), 
 70                   pollinterval=60 * 10, histmax=None): 
 71          """ 
 72          @type  p4port:       string 
 73          @param p4port:       p4 port definition (host:portno) 
 74          @type  p4user:       string 
 75          @param p4user:       p4 user 
 76          @type  p4passwd:     string 
 77          @param p4passwd:     p4 passwd 
 78          @type  p4base:       string 
 79          @param p4base:       p4 file specification to limit a poll to 
 80                               without the trailing '...' (i.e., //) 
 81          @type  p4bin:        string 
 82          @param p4bin:        path to p4 binary, defaults to just 'p4' 
 83          @type  split_file:   func 
 84          $param split_file:   splits a filename into branch and filename. 
 85          @type  pollinterval: int 
 86          @param pollinterval: interval in seconds between polls 
 87          @type  histmax:      int 
 88          @param histmax:      (obsolete) maximum number of changes to look back through. 
 89                               ignored; accepted for backwards compatibility. 
 90          """ 
 91   
 92          self.p4port = p4port 
 93          self.p4user = p4user 
 94          self.p4passwd = p4passwd 
 95          self.p4base = p4base 
 96          self.p4bin = p4bin 
 97          self.split_file = split_file 
 98          self.pollinterval = pollinterval 
 99          self.loop = LoopingCall(self.checkp4) 
100   
101 -    def startService(self): 
102          base.ChangeSource.startService(self) 
103   
104          # Don't start the loop just yet because the reactor isn't running. 
105          # Give it a chance to go and install our SIGCHLD handler before 
106          # spawning processes. 
107          reactor.callLater(0, self.loop.start, self.pollinterval) 
108   
109 -    def stopService(self): 
110          self.loop.stop() 
111          return base.ChangeSource.stopService(self) 
112   
113 -    def describe(self): 
114          return "p4source %s %s" % (self.p4port, self.p4base) 
115   
116 -    def checkp4(self): 
117          # Our return value is only used for unit testing. 
118          if self.working: 
119              log.msg("Skipping checkp4 because last one has not finished") 
120              return defer.succeed(None) 
121          else: 
122              self.working = True 
123              d = self._get_changes() 
124              d.addCallback(self._process_changes) 
125              d.addCallbacks(self._finished_ok, self._finished_failure) 
126              return d 
127   
128 -    def _finished_ok(self, res): 
129          assert self.working 
130          self.working = False 
131          return res 
132   
133 -    def _finished_failure(self, res): 
134          assert self.working 
135          self.working = False 
136   
137          # Again, the return value is only for unit testing. If there's a 
138          # failure, log it so it isn't lost. Use log.err to make sure unit 
139          # tests flunk if there was a problem. 
140          log.err(res, 'P4 poll failed') 
141          return None 
142   
143 -    def _get_process_output(self, args): 
144          env = dict([(e, os.environ.get(e)) for e in self.env_vars if os.environ.get(e)]) 
145          d = getProcessOutput(self.p4bin, args, env) 
146          return d 
147   
148 -    def _get_changes(self): 
149          args = [] 
150          if self.p4port: 
151              args.extend(['-p', self.p4port]) 
152          if self.p4user: 
153              args.extend(['-u', self.p4user]) 
154          if self.p4passwd: 
155              args.extend(['-P', self.p4passwd]) 
156          args.extend(['changes']) 
157          if self.last_change is not None: 
158              args.extend(['%s...@%d,now' % (self.p4base, self.last_change+1)]) 
159          else: 
160              args.extend(['-m', '1', '%s...' % (self.p4base,)]) 
161          return self._get_process_output(args) 
162   
163 -    def _process_changes(self, result): 
164          last_change = self.last_change 
165          changelists = [] 
166          for line in result.split('\n'): 
167              line = line.strip() 
168              if not line: continue 
169              m = self.changes_line_re.match(line) 
170              if not m: 
171                  raise P4PollerError("Unexpected 'p4 changes' output: %r" % result) 
172              num = int(m.group('num')) 
173              if last_change is None: 
174                  log.msg('P4Poller: starting at change %d' % num) 
175                  self.last_change = num 
176                  return [] 
177              changelists.append(num) 
178          changelists.reverse() # oldest first 
179   
180          # Retrieve each sequentially. 
181          d = defer.succeed(None) 
182          for c in changelists: 
183              d.addCallback(self._get_describe, c) 
184              d.addCallback(self._process_describe, c) 
185          return d 
186   
187 -    def _get_describe(self, dummy, num): 
188          args = [] 
189          if self.p4port: 
190              args.extend(['-p', self.p4port]) 
191          if self.p4user: 
192              args.extend(['-u', self.p4user]) 
193          if self.p4passwd: 
194              args.extend(['-P', self.p4passwd]) 
195          args.extend(['describe', '-s', str(num)]) 
196          return self._get_process_output(args) 
197   
198 -    def _process_describe(self, result, num): 
199          lines = result.split('\n') 
200          # SF#1555985: Wade Brainerd reports a stray ^M at the end of the date 
201          # field. The rstrip() is intended to remove that. 
202          lines[0] = lines[0].rstrip() 
203          m = self.describe_header_re.match(lines[0]) 
204          if not m: 
205              raise P4PollerError("Unexpected 'p4 describe -s' result: %r" % result) 
206          who = m.group('who') 
207          when = time.mktime(time.strptime(m.group('when'), self.datefmt)) 
208          comments = '' 
209          while not lines[0].startswith('Affected files'): 
210              comments += lines.pop(0) + '\n' 
211          lines.pop(0) # affected files 
212   
213          branch_files = {} # dict for branch mapped to file(s) 
214          while lines: 
215              line = lines.pop(0).strip() 
216              if not line: continue 
217              m = self.file_re.match(line) 
218              if not m: 
219                  raise P4PollerError("Invalid file line: %r" % line) 
220              path = m.group('path') 
221              if path.startswith(self.p4base): 
222                  branch, file = self.split_file(path[len(self.p4base):]) 
223                  if (branch == None and file == None): continue 
224                  if branch_files.has_key(branch): 
225                      branch_files[branch].append(file) 
226                  else: 
227                      branch_files[branch] = [file] 
228   
229          for branch in branch_files: 
230              c = changes.Change(who=who, 
231                                 files=branch_files[branch], 
232                                 comments=comments, 
233                                 revision=str(num), 
234                                 when=when, 
235                                 branch=branch) 
236              self.parent.addChange(c) 
237   
238          self.last_change = num 
239