buildbot.steps.source.git

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 from twisted.python import log, failure 17 from twisted.internet import defer 18 19 from buildbot.process import buildstep 20 from buildbot.steps.source import Source, _ComputeRepositoryURL 21 from buildbot.interfaces import BuildSlaveTooOldError

22 23 -class Git(Source):

24 """ Class for Git with all the smarts """ 25 name='git' 26 renderables = [ "repourl"] 27

28 - def __init__(self, repourl=None, branch='master', mode='incremental', 29 method=None, submodule=False, shallow=False, progress=False, 30 retryFetch=False, clobberOnFailure=False, **kwargs):

31 """ 32 @type repourl: string 33 @param repourl: the URL which points at the git repository 34 35 @type branch: string 36 @param branch: The branch or tag to check out by default. If 37 a build specifies a different branch, it will 38 be used instead of this. 39 40 @type submodules: boolean 41 @param submodules: Whether or not to update (and initialize) 42 git submodules. 43 44 @type mode: string 45 @param mode: Type of checkout. Described in docs. 46 47 @type method: string 48 @param method: Full builds can be done is different ways. This parameter 49 specifies which method to use. 50 51 @type progress: boolean 52 @param progress: Pass the --progress option when fetching. This 53 can solve long fetches getting killed due to 54 lack of output, but requires Git 1.7.2+. 55 @type shallow: boolean 56 @param shallow: Use a shallow or clone, if possible 57 58 @type retryFetch: boolean 59 @param retryFetch: Retry fetching before failing source checkout. 60 """ 61 62 self.branch = branch 63 self.method = method 64 self.prog = progress 65 self.repourl = repourl 66 self.retryFetch = retryFetch 67 self.submodule = submodule 68 self.shallow = shallow 69 self.fetchcount = 0 70 self.clobberOnFailure = clobberOnFailure 71 Source.__init__(self, **kwargs) 72 self.addFactoryArguments(branch=branch, 73 mode=mode, 74 method=method, 75 progress=progress, 76 repourl=repourl, 77 submodule=submodule, 78 shallow=shallow, 79 retryFetch=retryFetch, 80 clobberOnFailure= 81 clobberOnFailure, 82 ) 83 84 self.mode = mode 85 assert self.mode in ['incremental', 'full'] 86 assert self.repourl is not None 87 if self.mode == 'full': 88 assert self.method in ['clean', 'fresh', 'clobber', 'copy', None] 89 self.repourl = self.repourl and _ComputeRepositoryURL(self.repourl)

90

91 - def startVC(self, branch, revision, patch):

92 self.branch = branch or 'master' 93 self.revision = revision 94 self.method = self._getMethod() 95 self.stdio_log = self.addLog("stdio") 96 97 d = self.checkGit() 98 def checkInstall(gitInstalled): 99 if not gitInstalled: 100 raise BuildSlaveTooOldError("git is not installed on slave") 101 return 0

102 d.addCallback(checkInstall) 103 104 105 if self.mode == 'incremental': 106 d.addCallback(lambda _: self.incremental()) 107 elif self.mode == 'full': 108 d.addCallback(lambda _: self.full()) 109 d.addCallback(self.parseGotRevision) 110 d.addCallback(self.finish) 111 d.addErrback(self.failed) 112 return d

113 114 @defer.deferredGenerator

115 - def full(self):

116 if self.method == 'clobber': 117 wfd = defer.waitForDeferred(self.clobber()) 118 yield wfd 119 wfd.getResult() 120 return 121 elif self.method == 'copy': 122 wfd = defer.waitForDeferred(self.copy()) 123 yield wfd 124 wfd.getResult() 125 return 126 127 wfd = defer.waitForDeferred(self._sourcedirIsUpdatable()) 128 yield wfd 129 updatable = wfd.getResult() 130 if not updatable: 131 log.msg("No git repo present, making full clone") 132 d = self._doFull() 133 elif self.method == 'clean': 134 d = self.clean() 135 elif self.method == 'fresh': 136 d = self.fresh() 137 else: 138 raise ValueError("Unknown method, check your configuration") 139 wfd = defer.waitForDeferred(d) 140 yield wfd 141 wfd.getResult()

142

143 - def incremental(self):

144 d = self._sourcedirIsUpdatable() 145 def fetch(res): 146 # if revision exists checkout to that revision 147 # else fetch and update 148 if res == 0: 149 return self._dovccmd(['reset', '--hard', self.revision]) 150 else: 151 return self._doFetch(None)

152 153 def checkout(updatable): 154 if updatable: 155 if self.revision: 156 d = self._dovccmd(['cat-file', '-e', self.revision]) 157 else: 158 d = defer.succeed(1) 159 d.addCallback(fetch) 160 else: 161 d = self._doFull() 162 return d 163 164 d.addCallback(checkout) 165 d.addCallback(self._updateSubmodule) 166 return d 167

168 - def clean(self):

169 command = ['clean', '-f', '-d'] 170 d = self._dovccmd(command) 171 d.addCallback(self._doFetch) 172 d.addCallback(self._updateSubmodule) 173 d.addCallback(self._cleanSubmodule) 174 return d

175

176 - def clobber(self):

177 cmd = buildstep.LoggedRemoteCommand('rmdir', {'dir': self.workdir}) 178 cmd.useLog(self.stdio_log, False) 179 d = self.runCommand(cmd) 180 def checkRemoval(res): 181 if res != 0: 182 raise RuntimeError("Failed to delete directory") 183 return res

184 d.addCallback(lambda _: checkRemoval(cmd.rc)) 185 d.addCallback(lambda _: self._doFull()) 186 return d 187

188 - def fresh(self):

189 command = ['clean', '-f', '-d', '-x'] 190 d = self._dovccmd(command) 191 d.addCallback(self._doFetch) 192 d.addCallback(self._updateSubmodule) 193 d.addCallback(self._cleanSubmodule) 194 return d

195

196 - def copy(self):

197 cmd = buildstep.LoggedRemoteCommand('rmdir', {'dir': self.workdir}) 198 cmd.useLog(self.stdio_log, False) 199 d = self.runCommand(cmd) 200 201 self.workdir = 'source' 202 d.addCallback(lambda _: self.incremental()) 203 def copy(_): 204 cmd = buildstep.LoggedRemoteCommand('cpdir', 205 {'fromdir': 'source', 206 'todir':'build'}) 207 cmd.useLog(self.stdio_log, False) 208 d = self.runCommand(cmd) 209 return d

210 d.addCallback(copy) 211 def resetWorkdir(_): 212 self.workdir = 'build' 213 return 0 214 215 d.addCallback(resetWorkdir) 216 return d 217

218 - def finish(self, res):

219 d = defer.succeed(res) 220 def _gotResults(results): 221 self.setStatus(self.cmd, results) 222 log.msg("Closing log, sending result of the command %s " % \ 223 (self.cmd)) 224 return results

225 d.addCallback(_gotResults) 226 d.addCallbacks(self.finished, self.checkDisconnect) 227 return d 228

229 - def parseGotRevision(self, _):

230 d = self._dovccmd(['rev-parse', 'HEAD']) 231 def setrev(res): 232 revision = self.getLog('stdio').readlines()[-1].strip() 233 if len(revision) != 40: 234 raise failure.Failure 235 log.msg("Got Git revision %s" % (revision, )) 236 self.setProperty('got_revision', revision, 'Source') 237 return res

238 d.addCallback(setrev) 239 return d 240

241 - def _dovccmd(self, command, abandonOnFailure=True):

242 cmd = buildstep.RemoteShellCommand(self.workdir, ['git'] + command) 243 cmd.useLog(self.stdio_log, False) 244 log.msg("Starting git command : git %s" % (" ".join(command), )) 245 d = self.runCommand(cmd) 246 def evaluateCommand(cmd): 247 if abandonOnFailure and cmd.rc != 0: 248 log.msg("Source step failed while running command %s" % cmd) 249 raise failure.Failure(cmd.rc) 250 return cmd.rc

251 d.addCallback(lambda _: evaluateCommand(cmd)) 252 return d 253

254 - def _fetch(self, _):

255 command = ['fetch', '-t', self.repourl, self.branch] 256 # If the 'progress' option is set, tell git fetch to output 257 # progress information to the log. This can solve issues with 258 # long fetches killed due to lack of output, but only works 259 # with Git 1.7.2 or later. 260 if self.prog: 261 command.append('--progress') 262 263 d = self._dovccmd(command) 264 def checkout(_): 265 if self.revision: 266 rev = self.revision 267 else: 268 rev = 'FETCH_HEAD' 269 command = ['reset', '--hard', rev] 270 abandonOnFailure = not self.retryFetch and not self.clobberOnFailure 271 return self._dovccmd(command, abandonOnFailure)

272 d.addCallback(checkout) 273 return d 274 275 @defer.deferredGenerator

276 - def _doFetch(self, _):

277 """ 278 Handles fallbacks for failure of fetch, 279 wrapper for self._fetch 280 """ 281 wfd = defer.waitForDeferred(self._fetch(None)) 282 yield wfd 283 res = wfd.getResult() 284 if res == 0: 285 yield res 286 return 287 elif self.retryFetch: 288 d = self._fetch(None) 289 elif self.clobberOnFailure: 290 d = self.clobber() 291 else: 292 raise failure.Failure(res) 293 294 wfd = defer.waitForDeferred(d) 295 yield wfd 296 res = wfd.getResult()

297

298 - def _full(self):

299 if self.shallow: 300 command = ['clone', '--depth', '1', self.repourl, '.'] 301 else: 302 command = ['clone', self.repourl, '.'] 303 #Fix references 304 if self.prog: 305 command.append('--progress') 306 307 d = self._dovccmd(command, not self.clobberOnFailure) 308 # If revision specified checkout that revision 309 if self.revision: 310 d.addCallback(lambda _: self._dovccmd(['reset', '--hard', 311 self.revision], 312 not self.clobberOnFailure)) 313 # init and update submodules, recurisively. If there's not recursion 314 # it will not do it. 315 if self.submodule: 316 d.addCallback(lambda _: self._dovccmd(['submodule', 'update', 317 '--init', '--recursive'], 318 not self.clobberOnFailure)) 319 return d

320

321 - def _doFull(self):

322 d = self._full() 323 def clobber(res): 324 if res != 0: 325 if self.clobberOnFailure: 326 return self.clobber() 327 else: 328 raise failure.Failure(res) 329 else: 330 return res

331 d.addCallback(clobber) 332 return d 333

334 - def computeSourceRevision(self, changes):

335 if not changes: 336 return None 337 return changes[-1].revision

338

339 - def _sourcedirIsUpdatable(self):

340 cmd = buildstep.LoggedRemoteCommand('stat', {'file': self.workdir + '/.git'}) 341 log.msg(self.workdir) 342 cmd.useLog(self.stdio_log, False) 343 d = self.runCommand(cmd) 344 def _fail(tmp): 345 if cmd.rc != 0: 346 return False 347 return True

348 d.addCallback(_fail) 349 return d 350

351 - def _updateSubmodule(self, _):

352 if self.submodule: 353 return self._dovccmd(['submodule', 'update', '--recursive']) 354 else: 355 return defer.succeed(0)

356

357 - def _cleanSubmodule(self, _):

358 if self.submodule: 359 command = ['submodule', 'foreach', 'git', 'clean', '-f', '-d'] 360 if self.mode == 'full' and self.method == 'fresh': 361 command.append('-x') 362 return self._dovccmd(command) 363 else: 364 return defer.succeed(0)

365

366 - def _getMethod(self):

367 if self.method is not None and self.mode != 'incremental': 368 return self.method 369 elif self.mode == 'incremental': 370 return None 371 elif self.method is None and self.mode == 'full': 372 return 'fresh'

373

374 - def checkGit(self):

375 d = self._dovccmd(['--version']) 376 def check(res): 377 if res == 0: 378 return True 379 return False

380 d.addCallback(check) 381 return d 382

Source Code for Module buildbot.steps.source.git