Source Code for Module buildbot.changes.gitpoller

  1  # This file is part of Buildbot.  Buildbot is free software: you can 
  2  # redistribute it and/or modify it under the terms of the GNU General Public 
  3  # License as published by the Free Software Foundation, version 2. 
  4  # 
  5  # This program is distributed in the hope that it will be useful, but WITHOUT 
  6  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
  7  # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more 
  8  # details. 
  9  # 
 10  # You should have received a copy of the GNU General Public License along with 
 11  # this program; if not, write to the Free Software Foundation, Inc., 51 
 12  # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 
 13  # 
 14  # Copyright Buildbot Team Members 
 15   
 16  import time 
 17  import tempfile 
 18  import os 
 19   
 20  from twisted.python import log 
 21  from twisted.internet import defer, utils 
 22   
 23  from buildbot.changes import base, changes 
 24   
 25 -class GitPoller(base.PollingChangeSource): 
 26      """This source will poll a remote git repo for changes and submit 
 27      them to the change master.""" 
 28       
 29      compare_attrs = ["repourl", "branch", "workdir", 
 30                       "pollInterval", "gitbin", "usetimestamps", 
 31                       "category", "project"] 
 32                        
 33 -    def __init__(self, repourl, branch='master',  
 34                   workdir=None, pollInterval=10*60,  
 35                   gitbin='git', usetimestamps=True, 
 36                   category=None, project=None, 
 37                   pollinterval=-2): 
 38          # for backward compatibility; the parameter used to be spelled with 'i' 
 39          if pollinterval != -2: 
 40              pollInterval = pollinterval 
 41          if project is None: project = '' 
 42   
 43          self.repourl = repourl 
 44          self.branch = branch 
 45          self.pollInterval = pollInterval 
 46          self.lastChange = time.time() 
 47          self.lastPoll = time.time() 
 48          self.gitbin = gitbin 
 49          self.workdir = workdir 
 50          self.usetimestamps = usetimestamps 
 51          self.category = category 
 52          self.project = project 
 53          self.changeCount = 0 
 54          self.commitInfo  = {} 
 55           
 56          if self.workdir == None: 
 57              self.workdir = tempfile.gettempdir() + '/gitpoller_work' 
 58   
 59 -    def startService(self): 
 60          base.PollingChangeSource.startService(self) 
 61           
 62          if not os.path.exists(self.workdir): 
 63              log.msg('gitpoller: creating working dir %s' % self.workdir) 
 64              os.makedirs(self.workdir) 
 65               
 66          if not os.path.exists(self.workdir + r'/.git'): 
 67              log.msg('gitpoller: initializing working dir') 
 68              os.system(self.gitbin + ' clone ' + self.repourl + ' ' + self.workdir) 
 69           
 70 -    def describe(self): 
 71          status = "" 
 72          if not self.parent: 
 73              status = "[STOPPED - check log]" 
 74          str = 'GitPoller watching the remote git repository %s, branch: %s %s' \ 
 75                  % (self.repourl, self.branch, status) 
 76          return str 
 77   
 78 -    def poll(self): 
 79          d = self._get_changes() 
 80          d.addCallback(self._process_changes) 
 81          d.addErrback(self._process_changes_failure) 
 82          d.addCallback(self._catch_up) 
 83          d.addErrback(self._catch_up_failure) 
 84          return d 
 85   
 86 -    def _get_commit_comments(self, rev): 
 87          args = ['log', rev, '--no-walk', r'--format=%s%n%b'] 
 88          d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env={}, errortoo=False ) 
 89          d.addCallback(self._get_commit_comments_from_output) 
 90          return d 
 91   
 92 -    def _get_commit_comments_from_output(self,git_output): 
 93          stripped_output = git_output.strip() 
 94          if len(stripped_output) == 0: 
 95              raise EnvironmentError('could not get commit comment for rev') 
 96          self.commitInfo['comments'] = stripped_output 
 97          return self.commitInfo['comments'] # for tests 
 98   
 99 -    def _get_commit_timestamp(self, rev): 
100          # unix timestamp 
101          args = ['log', rev, '--no-walk', r'--format=%ct'] 
102          d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env={}, errortoo=False ) 
103          d.addCallback(self._get_commit_timestamp_from_output) 
104          return d 
105   
106 -    def _get_commit_timestamp_from_output(self, git_output): 
107          stripped_output = git_output.strip() 
108          if self.usetimestamps: 
109              try: 
110                  stamp = float(stripped_output) 
111              except Exception, e: 
112                      log.msg('gitpoller: caught exception converting output \'%s\' to timestamp' % stripped_output) 
113                      raise e 
114              self.commitInfo['timestamp'] = stamp 
115          else: 
116              self.commitInfo['timestamp'] = None 
117          return self.commitInfo['timestamp'] # for tests 
118   
119 -    def _get_commit_files(self, rev): 
120          args = ['log', rev, '--name-only', '--no-walk', r'--format=%n'] 
121          d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env={}, errortoo=False ) 
122          d.addCallback(self._get_commit_files_from_output) 
123          return d 
124   
125 -    def _get_commit_files_from_output(self, git_output): 
126          fileList = git_output.split() 
127          self.commitInfo['files'] = fileList 
128          return self.commitInfo['files'] # for tests 
129               
130 -    def _get_commit_name(self, rev): 
131          args = ['log', rev, '--no-walk', r'--format=%aE'] 
132          d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env={}, errortoo=False ) 
133          d.addCallback(self._get_commit_name_from_output) 
134          return d 
135   
136 -    def _get_commit_name_from_output(self, git_output): 
137          stripped_output = git_output.strip() 
138          if len(stripped_output) == 0: 
139              raise EnvironmentError('could not get commit name for rev') 
140          self.commitInfo['name'] = stripped_output 
141          return self.commitInfo['name'] # for tests 
142   
143 -    def _get_changes(self): 
144          log.msg('gitpoller: polling git repo at %s' % self.repourl) 
145   
146          self.lastPoll = time.time() 
147           
148          # get a deferred object that performs the fetch 
149          args = ['fetch', self.repourl, self.branch] 
150          # This command always produces data on stderr, but we actually do not care 
151          # about the stderr or stdout from this command. We set errortoo=True to 
152          # avoid an errback from the deferred. The callback which will be added to this 
153          # deferred will not use the response. 
154          d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env={}, errortoo=True ) 
155   
156          return d 
157   
158 -    def _process_changes(self, unused_output): 
159          # get the change list 
160          revListArgs = ['log', 'HEAD..FETCH_HEAD', r'--format=%H'] 
161          d = utils.getProcessOutput(self.gitbin, revListArgs, path=self.workdir, env={}, errortoo=False ) 
162          d.addCallback(self._process_changes_in_output) 
163          return d 
164       
165 -    def _process_changes_in_output(self, git_output): 
166          self.changeCount = 0 
167           
168          # process oldest change first 
169          revList = git_output.split() 
170          if revList: 
171              revList.reverse() 
172              self.changeCount = len(revList) 
173               
174          log.msg('gitpoller: processing %d changes: %s in "%s"' % (self.changeCount, revList, self.workdir) ) 
175   
176          for rev in revList: 
177              self.commitInfo = {} 
178   
179              deferreds = [ 
180                                  self._get_commit_timestamp(rev), 
181                                  self._get_commit_name(rev), 
182                                  self._get_commit_files(rev), 
183                                  self._get_commit_comments(rev), 
184                          ] 
185              dl = defer.DeferredList(deferreds) 
186              dl.addCallback(self._add_change,rev)         
187   
188   
189 -    def _add_change(self, results, rev): 
190          log.msg('gitpoller: _add_change results: "%s", rev: "%s" in "%s"' % (results, rev, self.workdir)) 
191   
192          c = changes.Change(who=self.commitInfo['name'], 
193                                 revision=rev, 
194                                 files=self.commitInfo['files'], 
195                                 comments=self.commitInfo['comments'], 
196                                 when=self.commitInfo['timestamp'], 
197                                 branch=self.branch, 
198                                 category=self.category, 
199                                 project=self.project, 
200                                 repository=self.repourl) 
201          log.msg('gitpoller: change "%s" in "%s"' % (c, self.workdir)) 
202          self.parent.addChange(c) 
203          self.lastChange = self.lastPoll 
204               
205   
206 -    def _process_changes_failure(self, f): 
207          log.msg('gitpoller: repo poll failed') 
208          log.err(f) 
209          # eat the failure to continue along the defered chain - we still want to catch up 
210          return None 
211           
212 -    def _catch_up(self, res): 
213          if self.changeCount == 0: 
214              log.msg('gitpoller: no changes, no catch_up') 
215              return 
216          log.msg('gitpoller: catching up to FETCH_HEAD') 
217          args = ['reset', '--hard', 'FETCH_HEAD'] 
218          d = utils.getProcessOutputAndValue(self.gitbin, args, path=self.workdir, env={}) 
219          def convert_nonzero_to_failure(res): 
220              (stdout, stderr, code) = res 
221              if code != 0: 
222                  raise EnvironmentError('catch up failed with exit code: %d' % code) 
223          d.addCallback(convert_nonzero_to_failure) 
224          return d 
225   
226 -    def _catch_up_failure(self, f): 
227          log.err(f) 
228          log.msg('gitpoller: please resolve issues in local repo: %s' % self.workdir) 
229          # this used to stop the service, but this is (a) unfriendly to tests and (b) 
230          # likely to leave the error message lost in a sea of other log messages 
231