buildbot.changes.gitpoller

17 """This source will poll a remote git repo for changes and submit 18 them to the change master.""" 19 20 compare_attrs = ["repourl", "branch", "workdir", 21 "pollinterval", "gitbin", "usetimestamps", 22 "category", "project"] 23 24 parent = None # filled in when we're added 25 loop = None 26 volatile = ['loop'] 27 working = False 28 running = False 29

30 - def __init__(self, repourl, branch='master', 31 workdir=None, pollinterval=10*60, 32 gitbin='git', usetimestamps=True, 33 category=None, project=''):

34 """ 35 @type repourl: string 36 @param repourl: the url that describes the remote repository, 37 e.g. git@example.com:foobaz/myrepo.git 38 39 @type branch: string 40 @param branch: the desired branch to fetch, will default to 'master' 41 42 @type workdir: string 43 @param workdir: the directory where the poller should keep its local repository. 44 will default to <tempdir>/gitpoller_work 45 46 @type pollinterval: int 47 @param pollinterval: interval in seconds between polls, default is 10 minutes 48 49 @type gitbin: string 50 @param gitbin: path to the git binary, defaults to just 'git' 51 52 @type usetimestamps: boolean 53 @param usetimestamps: parse each revision's commit timestamp (default True), or 54 ignore it in favor of the current time (to appear together 55 in the waterfall page) 56 57 @type category: string 58 @param category: catergory associated with the change. Attached to 59 the Change object produced by this changesource such that 60 it can be targeted by change filters. 61 62 @type project string 63 @param project project that the changes are associated to. Attached to 64 the Change object produced by this changesource such that 65 it can be targeted by change filters. 66 """ 67 68 self.repourl = repourl 69 self.branch = branch 70 self.pollinterval = pollinterval 71 self.lastChange = time.time() 72 self.lastPoll = time.time() 73 self.gitbin = gitbin 74 self.workdir = workdir 75 self.usetimestamps = usetimestamps 76 self.category = category 77 self.project = project 78 79 if self.workdir == None: 80 self.workdir = tempfile.gettempdir() + '/gitpoller_work'

81

82 - def startService(self):

83 self.loop = LoopingCall(self.poll) 84 base.ChangeSource.startService(self) 85 86 if not os.path.exists(self.workdir): 87 log.msg('gitpoller: creating working dir %s' % self.workdir) 88 os.makedirs(self.workdir) 89 90 if not os.path.exists(self.workdir + r'/.git'): 91 log.msg('gitpoller: initializing working dir') 92 os.system(self.gitbin + ' clone ' + self.repourl + ' ' + self.workdir) 93 94 reactor.callLater(0, self.loop.start, self.pollinterval) 95 96 self.running = True

97

98 - def stopService(self):

99 if self.running: 100 self.loop.stop() 101 self.running = False 102 return base.ChangeSource.stopService(self)

103

104 - def describe(self):

105 status = "" 106 if not self.running: 107 status = "[STOPPED - check log]" 108 str = 'GitPoller watching the remote git repository %s, branch: %s %s' \ 109 % (self.repourl, self.branch, status) 110 return str

111

112 - def poll(self):

113 if self.working: 114 log.msg('gitpoller: not polling git repo because last poll is still working') 115 else: 116 self.working = True 117 d = self._get_changes() 118 d.addCallback(self._process_changes) 119 d.addCallbacks(self._changes_finished_ok, self._changes_finished_failure) 120 d.addCallback(self._catch_up) 121 d.addCallbacks(self._catch_up_finished_ok, self._catch_up__finished_failure) 122 return

123

124 - def _get_git_output(self, args):

125 git_args = [self.gitbin] + args 126 127 p = subprocess.Popen(git_args, 128 cwd=self.workdir, 129 stdout=subprocess.PIPE) 130 131 # dirty hack - work around EINTR oddness on Mac builder 132 while True: 133 try: 134 output = p.communicate()[0] 135 break 136 except (OSError, select.error), e: 137 if e[0] == errno.EINTR: 138 continue 139 else: 140 raise 141 142 if p.returncode != 0: 143 raise EnvironmentError('call \'%s\' exited with error \'%s\', output: \'%s\'' % 144 (args, p.returncode, output)) 145 return output

146

147 - def _get_commit_comments(self, rev):

148 args = ['log', rev, '--no-walk', r'--format=%s%n%b'] 149 output = self._get_git_output(args) 150 151 if len(output.strip()) == 0: 152 raise EnvironmentError('could not get commit comment for rev %s' % rev) 153 154 return output

155

156 - def _get_commit_timestamp(self, rev):

157 # unix timestamp 158 args = ['log', rev, '--no-walk', r'--format=%ct'] 159 output = self._get_git_output(args) 160 161 try: 162 stamp = float(output) 163 except Exception, e: 164 log.msg('gitpoller: caught exception converting output \'%s\' to timestamp' % output) 165 raise e 166 167 return stamp

168

169 - def _get_commit_files(self, rev):

170 args = ['log', rev, '--name-only', '--no-walk', r'--format=%n'] 171 fileList = self._get_git_output(args).split() 172 return fileList

173

174 - def _get_commit_name(self, rev):

175 args = ['log', rev, '--no-walk', r'--format=%cn'] 176 output = self._get_git_output(args) 177 178 if len(output.strip()) == 0: 179 raise EnvironmentError('could not get commit name for rev %s' % rev) 180 181 return output

182

183 - def _get_changes(self):

184 log.msg('gitpoller: polling git repo at %s' % self.repourl) 185 186 self.lastPoll = time.time() 187 188 # get a deferred object that performs the fetch 189 args = ['fetch', self.repourl, self.branch] 190 d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env={}, errortoo=1 ) 191 192 return d

193

194 - def _process_changes(self, res):

195 # get the change list 196 revListArgs = ['log', 'HEAD..FETCH_HEAD', r'--format=%H'] 197 revs = self._get_git_output(revListArgs); 198 revCount = 0 199 200 # process oldest change first 201 revList = revs.split() 202 if revList: 203 revList.reverse() 204 revCount = len(revList) 205 206 log.msg('gitpoller: processing %d changes' % revCount ) 207 208 for rev in revList: 209 if self.usetimestamps: 210 commit_timestamp = self._get_commit_timestamp(rev) 211 else: 212 commit_timestamp = None # use current time 213 214 c = changes.Change(who = self._get_commit_name(rev), 215 revision = rev, 216 files = self._get_commit_files(rev), 217 comments = self._get_commit_comments(rev), 218 when = commit_timestamp, 219 branch = self.branch, 220 category = self.category, 221 project = self.project, 222 repository = self.repourl) 223 self.parent.addChange(c) 224 self.lastChange = self.lastPoll

225

226 - def _catch_up(self, res):

227 log.msg('gitpoller: catching up to FETCH_HEAD') 228 229 args = ['reset', '--hard', 'FETCH_HEAD'] 230 d = utils.getProcessOutputAndValue(self.gitbin, args, path=self.workdir, env={}) 231 return d;

232

233 - def _changes_finished_ok(self, res):

234 assert self.working 235 # check for failure -- this is probably never hit but the twisted docs 236 # are not clear enough to be sure. it is being kept "just in case" 237 if isinstance(res, failure.Failure): 238 return self._changes_finished_failure(res) 239 240 return res

241

242 - def _changes_finished_failure(self, res):

243 log.msg('gitpoller: repo poll failed: %s' % res) 244 assert self.working 245 # eat the failure to continue along the defered chain 246 # - we still want to catch up 247 return None

248

249 - def _catch_up_finished_ok(self, res):

250 assert self.working 251 252 # check for failure -- this is probably never hit but the twisted docs 253 # are not clear enough to be sure. it is being kept "just in case" 254 if isinstance(res, failure.Failure): 255 return self._catch_up__finished_failure(res) 256 257 elif isinstance(res, tuple): 258 (stdout, stderr, code) = res 259 if code != 0: 260 e = EnvironmentError('catch up failed with exit code: %d' % code) 261 return self._catch_up__finished_failure(failure.Failure(e)) 262 263 self.working = False 264 return res

265

266 - def _catch_up__finished_failure(self, res):

267 assert self.working 268 assert isinstance(res, failure.Failure) 269 self.working = False 270 271 log.msg('gitpoller: catch up failed: %s' % res) 272 log.msg('gitpoller: stopping service - please resolve issues in local repo: %s' % 273 self.workdir) 274 self.stopService() 275 return res

Source Code for Module buildbot.changes.gitpoller