Source Code for Module buildbot.changes.p4poller

  1  # This file is part of Buildbot.  Buildbot is free software: you can 
  2  # redistribute it and/or modify it under the terms of the GNU General Public 
  3  # License as published by the Free Software Foundation, version 2. 
  4  # 
  5  # This program is distributed in the hope that it will be useful, but WITHOUT 
  6  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
  7  # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more 
  8  # details. 
  9  # 
 10  # You should have received a copy of the GNU General Public License along with 
 11  # this program; if not, write to the Free Software Foundation, Inc., 51 
 12  # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 
 13  # 
 14  # Portions Copyright Buildbot Team Members 
 15  # Portions Copyright 2011 National Instruments 
 16   
 17   
 18  # Many thanks to Dave Peticolas for contributing this module 
 19   
 20  import re 
 21  import time 
 22  import os 
 23   
 24  from twisted.python import log 
 25  from twisted.internet import defer, utils 
 26   
 27  from buildbot import util 
 28  from buildbot.changes import base 
 29   
 30 -class P4PollerError(Exception): 
 31      """Something went wrong with the poll. This is used as a distinctive 
 32      exception type so that unit tests can detect and ignore it.""" 
 33   
 34 -def get_simple_split(branchfile): 
 35      """Splits the branchfile argument and assuming branch is  
 36         the first path component in branchfile, will return 
 37         branch and file else None.""" 
 38   
 39      index = branchfile.find('/') 
 40      if index == -1: return None, None 
 41      branch, file = branchfile.split('/', 1) 
 42      return branch, file 
 43   
 44 -class P4Source(base.PollingChangeSource, util.ComparableMixin): 
 45      """This source will poll a perforce repository for changes and submit 
 46      them to the change master.""" 
 47   
 48      compare_attrs = ["p4port", "p4user", "p4passwd", "p4base", 
 49                       "p4bin", "pollInterval"] 
 50   
 51      env_vars = ["P4CLIENT", "P4PORT", "P4PASSWD", "P4USER", 
 52                  "P4CHARSET"] 
 53   
 54      changes_line_re = re.compile( 
 55              r"Change (?P<num>\d+) on \S+ by \S+@\S+ '.*'$") 
 56      describe_header_re = re.compile( 
 57              r"Change \d+ by (?P<who>\S+)@\S+ on (?P<when>.+)$") 
 58      file_re = re.compile(r"^\.\.\. (?P<path>[^#]+)#\d+ [/\w]+$") 
 59      datefmt = '%Y/%m/%d %H:%M:%S' 
 60   
 61      parent = None # filled in when we're added 
 62      last_change = None 
 63      loop = None 
 64   
 65 -    def __init__(self, p4port=None, p4user=None, p4passwd=None, 
 66                   p4base='//', p4bin='p4', 
 67                   split_file=lambda branchfile: (None, branchfile), 
 68                   pollInterval=60 * 10, histmax=None, pollinterval=-2, 
 69                   encoding='utf8', project=None): 
 70          # for backward compatibility; the parameter used to be spelled with 'i' 
 71          if pollinterval != -2: 
 72              pollInterval = pollinterval 
 73   
 74          if project is None: 
 75              project = '' 
 76   
 77          self.p4port = p4port 
 78          self.p4user = p4user 
 79          self.p4passwd = p4passwd 
 80          self.p4base = p4base 
 81          self.p4bin = p4bin 
 82          self.split_file = split_file 
 83          self.pollInterval = pollInterval 
 84          self.encoding = encoding 
 85          self.project = project 
 86   
 87 -    def describe(self): 
 88          return "p4source %s %s" % (self.p4port, self.p4base) 
 89   
 90 -    def poll(self): 
 91          d = self._poll() 
 92          d.addErrback(log.err, 'P4 poll failed') 
 93          return d 
 94   
 95 -    def _get_process_output(self, args): 
 96          env = dict([(e, os.environ.get(e)) for e in self.env_vars if os.environ.get(e)]) 
 97          d = utils.getProcessOutput(self.p4bin, args, env) 
 98          return d 
 99   
100      @defer.deferredGenerator 
101 -    def _poll(self): 
102          args = [] 
103          if self.p4port: 
104              args.extend(['-p', self.p4port]) 
105          if self.p4user: 
106              args.extend(['-u', self.p4user]) 
107          if self.p4passwd: 
108              args.extend(['-P', self.p4passwd]) 
109          args.extend(['changes']) 
110          if self.last_change is not None: 
111              args.extend(['%s...@%d,now' % (self.p4base, self.last_change+1)]) 
112          else: 
113              args.extend(['-m', '1', '%s...' % (self.p4base,)]) 
114   
115          wfd = defer.waitForDeferred(self._get_process_output(args)) 
116          yield wfd 
117          result = wfd.getResult() 
118   
119          last_change = self.last_change 
120          changelists = [] 
121          for line in result.split('\n'): 
122              line = line.strip() 
123              if not line: continue 
124              m = self.changes_line_re.match(line) 
125              if not m: 
126                  raise P4PollerError("Unexpected 'p4 changes' output: %r" % result) 
127              num = int(m.group('num')) 
128              if last_change is None: 
129                  # first time through, the poller just gets a "baseline" for where to 
130                  # start on the next poll 
131                  log.msg('P4Poller: starting at change %d' % num) 
132                  self.last_change = num 
133                  return 
134              changelists.append(num) 
135          changelists.reverse() # oldest first 
136   
137          # Retrieve each sequentially. 
138          for num in changelists: 
139              args = [] 
140              if self.p4port: 
141                  args.extend(['-p', self.p4port]) 
142              if self.p4user: 
143                  args.extend(['-u', self.p4user]) 
144              if self.p4passwd: 
145                  args.extend(['-P', self.p4passwd]) 
146              args.extend(['describe', '-s', str(num)]) 
147              wfd = defer.waitForDeferred(self._get_process_output(args)) 
148              yield wfd 
149              result = wfd.getResult() 
150   
151              # decode the result from its designated encoding 
152              result = result.decode(self.encoding) 
153   
154              lines = result.split('\n') 
155              # SF#1555985: Wade Brainerd reports a stray ^M at the end of the date 
156              # field. The rstrip() is intended to remove that. 
157              lines[0] = lines[0].rstrip() 
158              m = self.describe_header_re.match(lines[0]) 
159              if not m: 
160                  raise P4PollerError("Unexpected 'p4 describe -s' result: %r" % result) 
161              who = m.group('who') 
162              when = time.mktime(time.strptime(m.group('when'), self.datefmt)) 
163              comments = '' 
164              while not lines[0].startswith('Affected files'): 
165                  comments += lines.pop(0) + '\n' 
166              lines.pop(0) # affected files 
167   
168              branch_files = {} # dict for branch mapped to file(s) 
169              while lines: 
170                  line = lines.pop(0).strip() 
171                  if not line: continue 
172                  m = self.file_re.match(line) 
173                  if not m: 
174                      raise P4PollerError("Invalid file line: %r" % line) 
175                  path = m.group('path') 
176                  if path.startswith(self.p4base): 
177                      branch, file = self.split_file(path[len(self.p4base):]) 
178                      if (branch == None and file == None): continue 
179                      if branch_files.has_key(branch): 
180                          branch_files[branch].append(file) 
181                      else: 
182                          branch_files[branch] = [file] 
183   
184              for branch in branch_files: 
185                  d = self.master.addChange( 
186                         author=who, 
187                         files=branch_files[branch], 
188                         comments=comments, 
189                         revision=str(num), 
190                         when_timestamp=util.epoch2datetime(when), 
191                         branch=branch, 
192                         project=self.project) 
193                  wfd = defer.waitForDeferred(d) 
194                  yield wfd 
195                  wfd.getResult() 
196   
197              self.last_change = num 
198