buildbot.steps.source.git

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 from twisted.python import log 17 from twisted.internet import defer 18 19 from buildbot.process import buildstep 20 from buildbot.steps.source import Source 21 from buildbot.interfaces import BuildSlaveTooOldError

22 23 -class Git(Source):

24 """ Class for Git with all the smarts """ 25 name='git' 26 renderables = [ "repourl"] 27

28 - def __init__(self, repourl=None, branch='HEAD', mode='incremental', 29 method=None, submodules=False, shallow=False, progress=False, 30 retryFetch=False, clobberOnFailure=False, **kwargs):

31 """ 32 @type repourl: string 33 @param repourl: the URL which points at the git repository 34 35 @type branch: string 36 @param branch: The branch or tag to check out by default. If 37 a build specifies a different branch, it will 38 be used instead of this. 39 40 @type submodules: boolean 41 @param submodules: Whether or not to update (and initialize) 42 git submodules. 43 44 @type mode: string 45 @param mode: Type of checkout. Described in docs. 46 47 @type method: string 48 @param method: Full builds can be done is different ways. This parameter 49 specifies which method to use. 50 51 @type progress: boolean 52 @param progress: Pass the --progress option when fetching. This 53 can solve long fetches getting killed due to 54 lack of output, but requires Git 1.7.2+. 55 @type shallow: boolean 56 @param shallow: Use a shallow or clone, if possible 57 58 @type retryFetch: boolean 59 @param retryFetch: Retry fetching before failing source checkout. 60 """ 61 62 self.branch = branch 63 self.method = method 64 self.prog = progress 65 self.repourl = repourl 66 self.retryFetch = retryFetch 67 self.submodules = submodules 68 self.shallow = shallow 69 self.fetchcount = 0 70 self.clobberOnFailure = clobberOnFailure 71 self.mode = mode 72 Source.__init__(self, **kwargs) 73 self.addFactoryArguments(branch=branch, 74 mode=mode, 75 method=method, 76 progress=progress, 77 repourl=repourl, 78 submodules=submodules, 79 shallow=shallow, 80 retryFetch=retryFetch, 81 clobberOnFailure= 82 clobberOnFailure, 83 ) 84 85 assert self.mode in ['incremental', 'full'] 86 assert self.repourl is not None 87 if self.mode == 'full': 88 assert self.method in ['clean', 'fresh', 'clobber', 'copy', None]

89

90 - def startVC(self, branch, revision, patch):

91 self.branch = branch or 'HEAD' 92 self.revision = revision 93 self.method = self._getMethod() 94 self.stdio_log = self.addLog("stdio") 95 96 d = self.checkGit() 97 def checkInstall(gitInstalled): 98 if not gitInstalled: 99 raise BuildSlaveTooOldError("git is not installed on slave") 100 return 0

101 d.addCallback(checkInstall) 102 103 if self.mode == 'incremental': 104 d.addCallback(lambda _: self.incremental()) 105 elif self.mode == 'full': 106 d.addCallback(lambda _: self.full()) 107 if patch: 108 d.addCallback(self.patch, patch) 109 d.addCallback(self.parseGotRevision) 110 d.addCallback(self.finish) 111 d.addErrback(self.failed) 112 return d

113 114 @defer.deferredGenerator

115 - def full(self):

116 if self.method == 'clobber': 117 wfd = defer.waitForDeferred(self.clobber()) 118 yield wfd 119 wfd.getResult() 120 return 121 elif self.method == 'copy': 122 wfd = defer.waitForDeferred(self.copy()) 123 yield wfd 124 wfd.getResult() 125 return 126 127 wfd = defer.waitForDeferred(self._sourcedirIsUpdatable()) 128 yield wfd 129 updatable = wfd.getResult() 130 if not updatable: 131 log.msg("No git repo present, making full clone") 132 d = self._doFull() 133 elif self.method == 'clean': 134 d = self.clean() 135 elif self.method == 'fresh': 136 d = self.fresh() 137 else: 138 raise ValueError("Unknown method, check your configuration") 139 wfd = defer.waitForDeferred(d) 140 yield wfd 141 wfd.getResult()

142 143 @defer.deferredGenerator

144 - def incremental(self):

145 wfd = defer.waitForDeferred( 146 self._sourcedirIsUpdatable()) 147 yield wfd 148 updatable = wfd.getResult() 149 150 # if not updateable, do a full checkout 151 if not updatable: 152 wfd = defer.waitForDeferred( 153 self._doFull()) 154 yield wfd 155 yield wfd.getResult() # return value 156 return 157 158 # test for existence of the revision; rc=1 indicates it does not exist 159 if self.revision: 160 wfd = defer.waitForDeferred( 161 self._dovccmd(['cat-file', '-e', self.revision], False)) 162 yield wfd 163 rc = wfd.getResult() 164 else: 165 rc = 1 166 167 # if revision exists checkout to that revision 168 # else fetch and update 169 if rc == 0: 170 wfd = defer.waitForDeferred( 171 self._dovccmd(['reset', '--hard', self.revision])) 172 yield wfd 173 wfd.getResult() 174 175 if self.branch != 'HEAD': 176 wfd = defer.waitForDeferred( 177 self._dovccmd(['branch', '-M', self.branch], abandonOnFailure=False)) 178 yield wfd 179 wfd.getResult() 180 else: 181 wfd = defer.waitForDeferred( 182 self._doFetch(None)) 183 yield wfd 184 wfd.getResult() 185 186 wfd = defer.waitForDeferred( 187 self._updateSubmodule(None)) 188 yield wfd 189 wfd.getResult()

190

191 - def clean(self):

192 command = ['clean', '-f', '-d'] 193 d = self._dovccmd(command) 194 d.addCallback(self._doFetch) 195 d.addCallback(self._updateSubmodule) 196 d.addCallback(self._cleanSubmodule) 197 return d

198

199 - def clobber(self):

200 cmd = buildstep.RemoteCommand('rmdir', {'dir': self.workdir, 201 'logEnviron': self.logEnviron,}) 202 cmd.useLog(self.stdio_log, False) 203 d = self.runCommand(cmd) 204 def checkRemoval(res): 205 if res != 0: 206 raise RuntimeError("Failed to delete directory") 207 return res

208 d.addCallback(lambda _: checkRemoval(cmd.rc)) 209 d.addCallback(lambda _: self._doFull()) 210 return d 211

212 - def fresh(self):

213 command = ['clean', '-f', '-d', '-x'] 214 d = self._dovccmd(command) 215 d.addCallback(self._doFetch) 216 d.addCallback(self._updateSubmodule) 217 d.addCallback(self._cleanSubmodule) 218 return d

219

220 - def copy(self):

221 cmd = buildstep.RemoteCommand('rmdir', {'dir': self.workdir, 222 'logEnviron': self.logEnviron,}) 223 cmd.useLog(self.stdio_log, False) 224 d = self.runCommand(cmd) 225 226 self.workdir = 'source' 227 d.addCallback(lambda _: self.incremental()) 228 def copy(_): 229 cmd = buildstep.RemoteCommand('cpdir', 230 {'fromdir': 'source', 231 'todir':'build', 232 'logEnviron': self.logEnviron,}) 233 cmd.useLog(self.stdio_log, False) 234 d = self.runCommand(cmd) 235 return d

236 d.addCallback(copy) 237 def resetWorkdir(_): 238 self.workdir = 'build' 239 return 0 240 241 d.addCallback(resetWorkdir) 242 return d 243

244 - def finish(self, res):

245 d = defer.succeed(res) 246 def _gotResults(results): 247 self.setStatus(self.cmd, results) 248 log.msg("Closing log, sending result of the command %s " % \ 249 (self.cmd)) 250 return results

251 d.addCallback(_gotResults) 252 d.addCallbacks(self.finished, self.checkDisconnect) 253 return d 254

255 - def parseGotRevision(self, _):

256 d = self._dovccmd(['rev-parse', 'HEAD'], collectStdout=True) 257 def setrev(stdout): 258 revision = stdout.strip() 259 if len(revision) != 40: 260 raise buildstep.BuildStepFailed() 261 log.msg("Got Git revision %s" % (revision, )) 262 self.setProperty('got_revision', revision, 'Source') 263 return 0

264 d.addCallback(setrev) 265 return d 266

267 - def _dovccmd(self, command, abandonOnFailure=True, collectStdout=False, extra_args={}):

268 cmd = buildstep.RemoteShellCommand(self.workdir, ['git'] + command, 269 env=self.env, 270 logEnviron=self.logEnviron, 271 collectStdout=collectStdout, 272 **extra_args) 273 cmd.useLog(self.stdio_log, False) 274 log.msg("Starting git command : git %s" % (" ".join(command), )) 275 d = self.runCommand(cmd) 276 def evaluateCommand(cmd): 277 if abandonOnFailure and cmd.rc != 0: 278 log.msg("Source step failed while running command %s" % cmd) 279 raise buildstep.BuildStepFailed() 280 if collectStdout: 281 return cmd.stdout 282 else: 283 return cmd.rc

284 d.addCallback(lambda _: evaluateCommand(cmd)) 285 return d 286

287 - def _fetch(self, _):

288 command = ['fetch', '-t', self.repourl, self.branch] 289 # If the 'progress' option is set, tell git fetch to output 290 # progress information to the log. This can solve issues with 291 # long fetches killed due to lack of output, but only works 292 # with Git 1.7.2 or later. 293 if self.prog: 294 command.append('--progress') 295 296 d = self._dovccmd(command) 297 def checkout(_): 298 if self.revision: 299 rev = self.revision 300 else: 301 rev = 'FETCH_HEAD' 302 command = ['reset', '--hard', rev] 303 abandonOnFailure = not self.retryFetch and not self.clobberOnFailure 304 return self._dovccmd(command, abandonOnFailure)

305 d.addCallback(checkout) 306 def renameBranch(res): 307 if res != 0: 308 return res 309 d = self._dovccmd(['branch', '-M', self.branch], abandonOnFailure=False) 310 # Ignore errors 311 d.addCallback(lambda _: res) 312 return d 313 314 if self.branch != 'HEAD': 315 d.addCallback(renameBranch) 316 return d 317

318 - def patch(self, _, patch):

319 d = self._dovccmd(['apply', '--index'], extra_args={'initial_stdin': patch}) 320 return d

321 322 @defer.deferredGenerator

323 - def _doFetch(self, _):

324 """ 325 Handles fallbacks for failure of fetch, 326 wrapper for self._fetch 327 """ 328 wfd = defer.waitForDeferred(self._fetch(None)) 329 yield wfd 330 res = wfd.getResult() 331 if res == 0: 332 yield res 333 return 334 elif self.retryFetch: 335 d = self._fetch(None) 336 elif self.clobberOnFailure: 337 d = self.clobber() 338 else: 339 raise buildstep.BuildStepFailed() 340 341 wfd = defer.waitForDeferred(d) 342 yield wfd 343 res = wfd.getResult()

344

345 - def _full(self):

346 if self.shallow: 347 command = ['clone', '--depth', '1', '--branch', self.branch, self.repourl, '.'] 348 else: 349 command = ['clone', '--branch', self.branch, self.repourl, '.'] 350 #Fix references 351 if self.prog: 352 command.append('--progress') 353 354 d = self._dovccmd(command, not self.clobberOnFailure) 355 # If revision specified checkout that revision 356 if self.revision: 357 d.addCallback(lambda _: self._dovccmd(['reset', '--hard', 358 self.revision], 359 not self.clobberOnFailure)) 360 # init and update submodules, recurisively. If there's not recursion 361 # it will not do it. 362 if self.submodules: 363 d.addCallback(lambda _: self._dovccmd(['submodule', 'update', 364 '--init', '--recursive'], 365 not self.clobberOnFailure)) 366 return d

367

368 - def _doFull(self):

369 d = self._full() 370 def clobber(res): 371 if res != 0: 372 if self.clobberOnFailure: 373 return self.clobber() 374 else: 375 raise buildstep.BuildStepFailed() 376 else: 377 return res

378 d.addCallback(clobber) 379 return d 380

381 - def computeSourceRevision(self, changes):

382 if not changes: 383 return None 384 return changes[-1].revision

385

386 - def _sourcedirIsUpdatable(self):

387 cmd = buildstep.RemoteCommand('stat', {'file': self.workdir + '/.git', 388 'logEnviron': self.logEnviron,}) 389 cmd.useLog(self.stdio_log, False) 390 d = self.runCommand(cmd) 391 def _fail(tmp): 392 if cmd.rc != 0: 393 return False 394 return True

395 d.addCallback(_fail) 396 return d 397

398 - def _updateSubmodule(self, _):

399 if self.submodules: 400 return self._dovccmd(['submodule', 'update', '--recursive']) 401 else: 402 return defer.succeed(0)

403

404 - def _cleanSubmodule(self, _):

405 if self.submodules: 406 command = ['submodule', 'foreach', 'git', 'clean', '-f', '-d'] 407 if self.mode == 'full' and self.method == 'fresh': 408 command.append('-x') 409 return self._dovccmd(command) 410 else: 411 return defer.succeed(0)

412

413 - def _getMethod(self):

414 if self.method is not None and self.mode != 'incremental': 415 return self.method 416 elif self.mode == 'incremental': 417 return None 418 elif self.method is None and self.mode == 'full': 419 return 'fresh'

420

421 - def checkGit(self):

422 d = self._dovccmd(['--version']) 423 def check(res): 424 if res == 0: 425 return True 426 return False

427 d.addCallback(check) 428 return d 429

Source Code for Module buildbot.steps.source.git