buildbot.changes.gitpoller

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 import time 17 import tempfile 18 import os 19 from twisted.python import log 20 from twisted.internet import defer, utils 21 22 from buildbot.util import deferredLocked 23 from buildbot.changes import base 24 from buildbot.util import epoch2datetime

25 26 -class GitPoller(base.PollingChangeSource):

27 """This source will poll a remote git repo for changes and submit 28 them to the change master.""" 29 30 compare_attrs = ["repourl", "branch", "workdir", 31 "pollInterval", "gitbin", "usetimestamps", 32 "category", "project"] 33

34 - def __init__(self, repourl, branch='master', 35 workdir=None, pollInterval=10*60, 36 gitbin='git', usetimestamps=True, 37 category=None, project=None, 38 pollinterval=-2, fetch_refspec=None, 39 encoding='utf-8'):

40 # for backward compatibility; the parameter used to be spelled with 'i' 41 if pollinterval != -2: 42 pollInterval = pollinterval 43 if project is None: project = '' 44 45 self.repourl = repourl 46 self.branch = branch 47 self.pollInterval = pollInterval 48 self.fetch_refspec = fetch_refspec 49 self.encoding = encoding 50 self.lastChange = time.time() 51 self.lastPoll = time.time() 52 self.gitbin = gitbin 53 self.workdir = workdir 54 self.usetimestamps = usetimestamps 55 self.category = category 56 self.project = project 57 self.changeCount = 0 58 self.commitInfo = {} 59 self.initLock = defer.DeferredLock() 60 61 if self.workdir == None: 62 self.workdir = tempfile.gettempdir() + '/gitpoller_work' 63 log.msg("WARNING: gitpoller using deprecated temporary workdir " + 64 "'%s'; consider setting workdir=" % self.workdir)

65

66 - def startService(self):

67 # make our workdir absolute, relative to the master's basedir 68 if not os.path.isabs(self.workdir): 69 self.workdir = os.path.join(self.master.basedir, self.workdir) 70 log.msg("gitpoller: using workdir '%s'" % self.workdir) 71 72 # initialize the repository we'll use to get changes; note that 73 # startService is not an event-driven method, so this method will 74 # instead acquire self.initLock immediately when it is called. 75 if not os.path.exists(self.workdir + r'/.git'): 76 d = self.initRepository() 77 d.addErrback(log.err, 'while initializing GitPoller repository') 78 else: 79 log.msg("GitPoller repository already exists") 80 81 # call this *after* initRepository, so that the initLock is locked first 82 base.PollingChangeSource.startService(self)

83 84 @deferredLocked('initLock')

85 - def initRepository(self):

86 d = defer.succeed(None) 87 def make_dir(_): 88 dirpath = os.path.dirname(self.workdir.rstrip(os.sep)) 89 if not os.path.exists(dirpath): 90 log.msg('gitpoller: creating parent directories for workdir') 91 os.makedirs(dirpath)

92 d.addCallback(make_dir) 93 94 def git_init(_): 95 log.msg('gitpoller: initializing working dir from %s' % self.repourl) 96 d = utils.getProcessOutputAndValue(self.gitbin, 97 ['init', self.workdir], env=os.environ) 98 d.addCallback(self._convert_nonzero_to_failure) 99 d.addErrback(self._stop_on_failure) 100 return d

101 d.addCallback(git_init) 102 103 def git_remote_add(_): 104 d = utils.getProcessOutputAndValue(self.gitbin, 105 ['remote', 'add', 'origin', self.repourl], 106 path=self.workdir, env=os.environ) 107 d.addCallback(self._convert_nonzero_to_failure) 108 d.addErrback(self._stop_on_failure) 109 return d 110 d.addCallback(git_remote_add) 111 112 def git_fetch_origin(_): 113 args = ['fetch', 'origin'] 114 self._extend_with_fetch_refspec(args) 115 d = utils.getProcessOutputAndValue(self.gitbin, args, 116 path=self.workdir, env=os.environ) 117 d.addCallback(self._convert_nonzero_to_failure) 118 d.addErrback(self._stop_on_failure) 119 return d 120 d.addCallback(git_fetch_origin) 121 122 def set_master(_): 123 log.msg('gitpoller: checking out %s' % self.branch) 124 if self.branch == 'master': # repo is already on branch 'master', so reset 125 d = utils.getProcessOutputAndValue(self.gitbin, 126 ['reset', '--hard', 'origin/%s' % self.branch], 127 path=self.workdir, env=os.environ) 128 else: 129 d = utils.getProcessOutputAndValue(self.gitbin, 130 ['checkout', '-b', self.branch, 'origin/%s' % self.branch], 131 path=self.workdir, env=os.environ) 132 d.addCallback(self._convert_nonzero_to_failure) 133 d.addErrback(self._stop_on_failure) 134 return d 135 d.addCallback(set_master) 136 def get_rev(_): 137 d = utils.getProcessOutputAndValue(self.gitbin, 138 ['rev-parse', self.branch], 139 path=self.workdir, env={}) 140 d.addCallback(self._convert_nonzero_to_failure) 141 d.addErrback(self._stop_on_failure) 142 d.addCallback(lambda (out, err, code) : out.strip()) 143 return d 144 d.addCallback(get_rev) 145 def print_rev(rev): 146 log.msg("gitpoller: finished initializing working dir from %s at rev %s" 147 % (self.repourl, rev)) 148 d.addCallback(print_rev) 149 return d 150

151 - def describe(self):

152 status = "" 153 if not self.master: 154 status = "[STOPPED - check log]" 155 str = 'GitPoller watching the remote git repository %s, branch: %s %s' \ 156 % (self.repourl, self.branch, status) 157 return str

158 159 @deferredLocked('initLock')

160 - def poll(self):

161 d = self._get_changes() 162 d.addCallback(self._process_changes) 163 d.addErrback(self._process_changes_failure) 164 d.addCallback(self._catch_up) 165 d.addErrback(self._catch_up_failure) 166 return d

167

168 - def _get_commit_comments(self, rev):

169 args = ['log', rev, '--no-walk', r'--format=%s%n%b'] 170 d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env=os.environ, errortoo=False ) 171 def process(git_output): 172 stripped_output = git_output.strip().decode(self.encoding) 173 if len(stripped_output) == 0: 174 raise EnvironmentError('could not get commit comment for rev') 175 return stripped_output

176 d.addCallback(process) 177 return d 178

179 - def _get_commit_timestamp(self, rev):

180 # unix timestamp 181 args = ['log', rev, '--no-walk', r'--format=%ct'] 182 d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env=os.environ, errortoo=False ) 183 def process(git_output): 184 stripped_output = git_output.strip() 185 if self.usetimestamps: 186 try: 187 stamp = float(stripped_output) 188 except Exception, e: 189 log.msg('gitpoller: caught exception converting output \'%s\' to timestamp' % stripped_output) 190 raise e 191 return stamp 192 else: 193 return None

194 d.addCallback(process) 195 return d 196

197 - def _get_commit_files(self, rev):

198 args = ['log', rev, '--name-only', '--no-walk', r'--format=%n'] 199 d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env=os.environ, errortoo=False ) 200 def process(git_output): 201 fileList = git_output.split() 202 return fileList

203 d.addCallback(process) 204 return d 205

206 - def _get_commit_author(self, rev):

207 args = ['log', rev, '--no-walk', r'--format=%aN <%aE>'] 208 d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env=os.environ, errortoo=False ) 209 def process(git_output): 210 stripped_output = git_output.strip().decode(self.encoding) 211 if len(stripped_output) == 0: 212 raise EnvironmentError('could not get commit author for rev') 213 return stripped_output

214 d.addCallback(process) 215 return d 216

217 - def _get_changes(self):

218 log.msg('gitpoller: polling git repo at %s' % self.repourl) 219 220 self.lastPoll = time.time() 221 222 # get a deferred object that performs the fetch 223 args = ['fetch', 'origin'] 224 self._extend_with_fetch_refspec(args) 225 226 # This command always produces data on stderr, but we actually do not care 227 # about the stderr or stdout from this command. We set errortoo=True to 228 # avoid an errback from the deferred. The callback which will be added to this 229 # deferred will not use the response. 230 d = utils.getProcessOutput(self.gitbin, args, 231 path=self.workdir, 232 env=os.environ, errortoo=True ) 233 234 return d

235 236 @defer.deferredGenerator

237 - def _process_changes(self, unused_output):

238 # get the change list 239 revListArgs = ['log', '%s..origin/%s' % (self.branch, self.branch), r'--format=%H'] 240 self.changeCount = 0 241 d = utils.getProcessOutput(self.gitbin, revListArgs, path=self.workdir, 242 env=os.environ, errortoo=False ) 243 wfd = defer.waitForDeferred(d) 244 yield wfd 245 results = wfd.getResult() 246 247 # process oldest change first 248 revList = results.split() 249 if not revList: 250 return 251 252 revList.reverse() 253 self.changeCount = len(revList) 254 255 log.msg('gitpoller: processing %d changes: %s in "%s"' 256 % (self.changeCount, revList, self.workdir) ) 257 258 for rev in revList: 259 dl = defer.DeferredList([ 260 self._get_commit_timestamp(rev), 261 self._get_commit_author(rev), 262 self._get_commit_files(rev), 263 self._get_commit_comments(rev), 264 ], consumeErrors=True) 265 266 wfd = defer.waitForDeferred(dl) 267 yield wfd 268 results = wfd.getResult() 269 270 # check for failures 271 failures = [ r[1] for r in results if not r[0] ] 272 if failures: 273 # just fail on the first error; they're probably all related! 274 raise failures[0] 275 276 timestamp, author, files, comments = [ r[1] for r in results ] 277 d = self.master.addChange( 278 author=author, 279 revision=rev, 280 files=files, 281 comments=comments, 282 when_timestamp=epoch2datetime(timestamp), 283 branch=self.branch, 284 category=self.category, 285 project=self.project, 286 repository=self.repourl, 287 src='git') 288 wfd = defer.waitForDeferred(d) 289 yield wfd 290 results = wfd.getResult()

291

292 - def _process_changes_failure(self, f):

293 log.msg('gitpoller: repo poll failed') 294 log.err(f) 295 # eat the failure to continue along the defered chain - we still want to catch up 296 return None

297

298 - def _catch_up(self, res):

299 if self.changeCount == 0: 300 log.msg('gitpoller: no changes, no catch_up') 301 return 302 log.msg('gitpoller: catching up tracking branch') 303 args = ['reset', '--hard', 'origin/%s' % (self.branch,)] 304 d = utils.getProcessOutputAndValue(self.gitbin, args, path=self.workdir, env=os.environ) 305 d.addCallback(self._convert_nonzero_to_failure) 306 return d

307

308 - def _catch_up_failure(self, f):

309 log.err(f) 310 log.msg('gitpoller: please resolve issues in local repo: %s' % self.workdir)

311 # this used to stop the service, but this is (a) unfriendly to tests and (b) 312 # likely to leave the error message lost in a sea of other log messages 313

314 - def _convert_nonzero_to_failure(self, res):

315 "utility method to handle the result of getProcessOutputAndValue" 316 (stdout, stderr, code) = res 317 if code != 0: 318 raise EnvironmentError('command failed with exit code %d: %s' % (code, stderr)) 319 return (stdout, stderr, code)

320

321 - def _stop_on_failure(self, f):

322 "utility method to stop the service when a failure occurs" 323 if self.running: 324 d = defer.maybeDeferred(lambda : self.stopService()) 325 d.addErrback(log.err, 'while stopping broken GitPoller service') 326 return f

327

328 - def _extend_with_fetch_refspec(self, args):

329 if self.fetch_refspec: 330 if type(self.fetch_refspec) in (list,set): 331 args.extend(self.fetch_refspec) 332 else: 333 args.append(self.fetch_refspec)

334

Source Code for Module buildbot.changes.gitpoller