buildbot.changes.gitpoller

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 import os 17 import urllib 18 from twisted.python import log 19 from twisted.internet import defer, utils 20 21 from buildbot.changes import base 22 from buildbot.util import epoch2datetime 23 from buildbot.util.state import StateMixin 24 from buildbot import config

25 26 -class GitPoller(base.PollingChangeSource, StateMixin):

27 """This source will poll a remote git repo for changes and submit 28 them to the change master.""" 29 30 compare_attrs = ["repourl", "branches", "workdir", 31 "pollInterval", "gitbin", "usetimestamps", 32 "category", "project"] 33

34 - def __init__(self, repourl, branches=None, branch=None, 35 workdir=None, pollInterval=10*60, 36 gitbin='git', usetimestamps=True, 37 category=None, project=None, 38 pollinterval=-2, fetch_refspec=None, 39 encoding='utf-8'):

40 41 # for backward compatibility; the parameter used to be spelled with 'i' 42 if pollinterval != -2: 43 pollInterval = pollinterval 44 45 base.PollingChangeSource.__init__(self, name=repourl, 46 pollInterval=pollInterval) 47 48 if project is None: project = '' 49 50 if branch and branches: 51 config.error("GitPoller: can't specify both branch and branches") 52 elif branch: 53 branches = [branch] 54 elif not branches: 55 branches = ['master'] 56 57 self.repourl = repourl 58 self.branches = branches 59 self.encoding = encoding 60 self.gitbin = gitbin 61 self.workdir = workdir 62 self.usetimestamps = usetimestamps 63 self.category = category 64 self.project = project 65 self.changeCount = 0 66 self.lastRev = {} 67 68 if fetch_refspec is not None: 69 config.error("GitPoller: fetch_refspec is no longer supported. " 70 "Instead, only the given branches are downloaded.") 71 72 if self.workdir == None: 73 self.workdir = 'gitpoller-work'

74

75 - def startService(self):

76 # make our workdir absolute, relative to the master's basedir 77 if not os.path.isabs(self.workdir): 78 self.workdir = os.path.join(self.master.basedir, self.workdir) 79 log.msg("gitpoller: using workdir '%s'" % self.workdir) 80 81 d = self.getState('lastRev', {}) 82 def setLastRev(lastRev): 83 self.lastRev = lastRev

84 d.addCallback(setLastRev) 85 86 d.addCallback(lambda _: 87 base.PollingChangeSource.startService(self)) 88 d.addErrback(log.err, 'while initializing GitPoller repository') 89 90 return d

91

92 - def describe(self):

93 status = "" 94 if not self.master: 95 status = "[STOPPED - check log]" 96 str = ('GitPoller watching the remote git repository %s, branches: %s %s' 97 % (self.repourl, ', '.join(self.branches), status)) 98 return str

99 100 @defer.inlineCallbacks

101 - def poll(self):

102 yield self._dovccmd('init', ['--bare', self.workdir]) 103 104 refspecs = [ 105 '+%s:%s'% (branch, self._localBranch(branch)) 106 for branch in self.branches 107 ] 108 yield self._dovccmd('fetch', 109 [self.repourl] + refspecs, path=self.workdir) 110 111 revs = {} 112 for branch in self.branches: 113 try: 114 revs[branch] = rev = yield self._dovccmd('rev-parse', 115 [self._localBranch(branch)], path=self.workdir) 116 yield self._process_changes(rev, branch) 117 except: 118 log.err(_why="trying to poll branch %s of %s" 119 % (branch, self.repourl)) 120 121 self.lastRev.update(revs) 122 yield self.setState('lastRev', self.lastRev)

123

124 - def _get_commit_comments(self, rev):

125 args = ['--no-walk', r'--format=%s%n%b', rev, '--'] 126 d = self._dovccmd('log', args, path=self.workdir) 127 def process(git_output): 128 git_output = git_output.decode(self.encoding) 129 if len(git_output) == 0: 130 raise EnvironmentError('could not get commit comment for rev') 131 return git_output

132 d.addCallback(process) 133 return d 134

135 - def _get_commit_timestamp(self, rev):

136 # unix timestamp 137 args = ['--no-walk', r'--format=%ct', rev, '--'] 138 d = self._dovccmd('log', args, path=self.workdir) 139 def process(git_output): 140 if self.usetimestamps: 141 try: 142 stamp = float(git_output) 143 except Exception, e: 144 log.msg('gitpoller: caught exception converting output \'%s\' to timestamp' % git_output) 145 raise e 146 return stamp 147 else: 148 return None

149 d.addCallback(process) 150 return d 151

152 - def _get_commit_files(self, rev):

153 args = ['--name-only', '--no-walk', r'--format=%n', rev, '--'] 154 d = self._dovccmd('log', args, path=self.workdir) 155 def process(git_output): 156 fileList = git_output.split() 157 return fileList

158 d.addCallback(process) 159 return d 160

161 - def _get_commit_author(self, rev):

162 args = ['--no-walk', r'--format=%aN <%aE>', rev, '--'] 163 d = self._dovccmd('log', args, path=self.workdir) 164 def process(git_output): 165 git_output = git_output.decode(self.encoding) 166 if len(git_output) == 0: 167 raise EnvironmentError('could not get commit author for rev') 168 return git_output

169 d.addCallback(process) 170 return d 171 172 @defer.inlineCallbacks

173 - def _process_changes(self, newRev, branch):

174 """ 175 Read changes since last change. 176 177 - Read list of commit hashes. 178 - Extract details from each commit. 179 - Add changes to database. 180 """ 181 182 lastRev = self.lastRev.get(branch) 183 self.lastRev[branch] = newRev 184 if not lastRev: 185 return 186 187 # get the change list 188 revListArgs = [r'--format=%H', '%s..%s' % (lastRev, newRev), '--'] 189 self.changeCount = 0 190 results = yield self._dovccmd('log', revListArgs, path=self.workdir) 191 192 193 # process oldest change first 194 revList = results.split() 195 revList.reverse() 196 self.changeCount = len(revList) 197 198 log.msg('gitpoller: processing %d changes: %s from "%s"' 199 % (self.changeCount, revList, self.repourl) ) 200 201 for rev in revList: 202 dl = defer.DeferredList([ 203 self._get_commit_timestamp(rev), 204 self._get_commit_author(rev), 205 self._get_commit_files(rev), 206 self._get_commit_comments(rev), 207 ], consumeErrors=True) 208 209 results = yield dl 210 211 # check for failures 212 failures = [ r[1] for r in results if not r[0] ] 213 if failures: 214 # just fail on the first error; they're probably all related! 215 raise failures[0] 216 217 timestamp, author, files, comments = [ r[1] for r in results ] 218 yield self.master.addChange( 219 author=author, 220 revision=rev, 221 files=files, 222 comments=comments, 223 when_timestamp=epoch2datetime(timestamp), 224 branch=branch, 225 category=self.category, 226 project=self.project, 227 repository=self.repourl, 228 src='git')

229

230 - def _dovccmd(self, command, args, path=None):

231 d = utils.getProcessOutputAndValue(self.gitbin, 232 [command] + args, path=path, env=os.environ) 233 def _convert_nonzero_to_failure(res): 234 "utility to handle the result of getProcessOutputAndValue" 235 (stdout, stderr, code) = res 236 if code != 0: 237 raise EnvironmentError('command failed with exit code %d: %s' 238 % (code, stderr)) 239 return stdout.strip()

240 d.addCallback(_convert_nonzero_to_failure) 241 return d 242

243 - def _localBranch(self, branch):

244 return "refs/buildbot/%s/%s" % (urllib.quote(self.repourl, ''), branch)

245

Source Code for Module buildbot.changes.gitpoller