buildbot.steps.source.svn

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 import re 17 import xml.dom.minidom 18 import xml.parsers.expat 19 20 from twisted.python import log 21 from twisted.internet import defer 22 23 from buildbot.process import buildstep 24 from buildbot.steps.source import Source 25 from buildbot.interfaces import BuildSlaveTooOldError 26 from buildbot.config import ConfigErrors

27 28 -class SVN(Source):

29 """I perform Subversion checkout/update operations.""" 30 31 name = 'svn' 32 branch_placeholder = '%%BRANCH%%' 33 34 renderables = [ 'repourl', 'baseURL' ] 35 possible_modes = ('incremental', 'full') 36 possible_methods = ('clean', 'fresh', 'clobber', 'copy', 'export', None) 37

38 - def __init__(self, repourl=None, baseURL=None, mode='incremental', 39 method=None, defaultBranch=None, username=None, 40 password=None, extra_args=None, keep_on_purge=None, 41 depth=None, **kwargs):

42 43 self.repourl = repourl 44 self.baseURL = baseURL 45 self.branch = defaultBranch 46 self.username = username 47 self.password = password 48 self.extra_args = extra_args 49 self.keep_on_purge = keep_on_purge or [] 50 self.depth = depth 51 self.method=method 52 self.mode = mode 53 Source.__init__(self, **kwargs) 54 self.addFactoryArguments(repourl=repourl, 55 baseURL=baseURL, 56 mode=mode, 57 method=method, 58 defaultBranch=defaultBranch, 59 password=password, 60 username=username, 61 extra_args=extra_args, 62 keep_on_purge=keep_on_purge, 63 depth=depth, 64 ) 65 errors = [] 66 if self.mode not in self.possible_modes: 67 errors.append("mode %s is not one of %s" % (self.mode, self.possible_modes)) 68 if self.method not in self.possible_methods: 69 errors.append("method %s is not one of %s" % (self.method, self.possible_methods)) 70 71 if repourl and baseURL: 72 errors.append("you must provide exactly one of repourl and baseURL") 73 74 if repourl is None and baseURL is None: 75 errors.append("you must privide at least one of repourl and baseURL") 76 if errors: 77 raise ConfigErrors(errors)

78

79 - def startVC(self, branch, revision, patch):

80 self.revision = revision 81 self.method = self._getMethod() 82 self.repourl = self.getRepoUrl(branch) 83 self.stdio_log = self.addLog("stdio") 84 85 d = self.checkSvn() 86 def checkInstall(svnInstalled): 87 if not svnInstalled: 88 raise BuildSlaveTooOldError("SVN is not installed on slave") 89 return 0

90 91 if self.mode == 'full': 92 d.addCallback(self.full) 93 elif self.mode == 'incremental': 94 d.addCallback(self.incremental) 95 d.addCallback(self.parseGotRevision) 96 d.addCallback(self.finish) 97 d.addErrback(self.failed) 98 return d

99 100 @defer.deferredGenerator

101 - def full(self, _):

102 if self.method == 'clobber': 103 wfd = defer.waitForDeferred(self.clobber()) 104 yield wfd 105 wfd.getResult() 106 return 107 elif self.method in ['copy', 'export']: 108 wfd = defer.waitForDeferred(self.copy()) 109 yield wfd 110 wfd.getResult() 111 return 112 113 wfd = defer.waitForDeferred(self._sourcedirIsUpdatable()) 114 yield wfd 115 updatable = wfd.getResult() 116 if not updatable: 117 # blow away the old (un-updatable) directory 118 wfd = defer.waitForDeferred( 119 self._rmdir(self.workdir)) 120 yield wfd 121 wfd.getResult() 122 123 # then do a checkout 124 checkout_cmd = ['checkout', self.repourl, '.'] 125 if self.revision: 126 checkout_cmd.extend(["--revision", str(self.revision)]) 127 wfd = defer.waitForDeferred( 128 self._dovccmd(checkout_cmd)) 129 yield wfd 130 wfd.getResult() 131 elif self.method == 'clean': 132 wfd = defer.waitForDeferred( 133 self.clean()) 134 yield wfd 135 wfd.getResult() 136 elif self.method == 'fresh': 137 wfd = defer.waitForDeferred( 138 self.fresh()) 139 yield wfd 140 wfd.getResult()

141 142 @defer.deferredGenerator

143 - def incremental(self, _):

144 wfd = defer.waitForDeferred( 145 self._sourcedirIsUpdatable()) 146 yield wfd 147 updatable = wfd.getResult() 148 149 if not updatable: 150 # blow away the old (un-updatable) directory 151 wfd = defer.waitForDeferred( 152 self._rmdir(self.workdir)) 153 yield wfd 154 wfd.getResult() 155 156 # and plan to do a checkout 157 command = ['checkout', self.repourl, '.'] 158 else: 159 # otherwise, do an update 160 command = ['update'] 161 if self.revision: 162 command.extend(['--revision', str(self.revision)]) 163 164 wfd = defer.waitForDeferred( 165 self._dovccmd(command)) 166 yield wfd 167 wfd.getResult()

168 169 @defer.deferredGenerator

170 - def clobber(self):

171 cmd = buildstep.RemoteCommand('rmdir', {'dir': self.workdir, 172 'logEnviron': self.logEnviron,}) 173 cmd.useLog(self.stdio_log, False) 174 wfd = defer.waitForDeferred( 175 self.runCommand(cmd)) 176 yield wfd 177 wfd.getResult() 178 if cmd.rc != 0: 179 raise buildstep.BuildStepFailed() 180 181 checkout_cmd = ['checkout', self.repourl, '.'] 182 if self.revision: 183 checkout_cmd.extend(["--revision", str(self.revision)]) 184 185 wfd = defer.waitForDeferred( 186 self._dovccmd(checkout_cmd)) 187 yield wfd 188 wfd.getResult()

189

190 - def fresh(self):

191 d = self.purge(True) 192 cmd = ['update'] 193 if self.revision: 194 cmd.extend(['--revision', str(self.revision)]) 195 d.addCallback(lambda _: self._dovccmd(cmd)) 196 return d

197

198 - def clean(self):

199 d = self.purge(False) 200 cmd = ['update'] 201 if self.revision: 202 cmd.extend(['--revision', str(self.revision)]) 203 d.addCallback(lambda _: self._dovccmd(cmd)) 204 return d

205 206 @defer.deferredGenerator

207 - def copy(self):

208 cmd = buildstep.RemoteCommand('rmdir', {'dir': self.workdir, 209 'logEnviron': self.logEnviron,}) 210 cmd.useLog(self.stdio_log, False) 211 wfd = defer.waitForDeferred( 212 self.runCommand(cmd)) 213 yield wfd 214 wfd.getResult() 215 216 if cmd.rc != 0: 217 raise buildstep.BuildStepFailed() 218 219 # temporarily set workdir = 'source' and do an incremental checkout 220 try: 221 old_workdir = self.workdir 222 self.workdir = 'source' 223 wfd = defer.waitForDeferred( 224 self.incremental(None)) 225 yield wfd 226 wfd.getResult() 227 except: # finally doesn't work in python-2.4 228 self.workdir = old_workdir 229 raise 230 self.workdir = old_workdir 231 232 # if we're copying, copy; otherwise, export from source to build 233 if self.method == 'copy': 234 cmd = buildstep.RemoteCommand('cpdir', 235 { 'fromdir': 'source', 'todir':self.workdir, 236 'logEnviron': self.logEnviron }) 237 else: 238 export_cmd = ['svn', 'export'] 239 if self.revision: 240 export_cmd.extend(["--revision", str(self.revision)]) 241 export_cmd.extend(['source', self.workdir]) 242 243 cmd = buildstep.RemoteShellCommand('', export_cmd, 244 env=self.env, logEnviron=self.logEnviron) 245 cmd.useLog(self.stdio_log, False) 246 247 wfd = defer.waitForDeferred( 248 self.runCommand(cmd)) 249 yield wfd 250 wfd.getResult() 251 252 if cmd.rc != 0: 253 raise buildstep.BuildStepFailed()

254

255 - def finish(self, res):

256 d = defer.succeed(res) 257 def _gotResults(results): 258 self.setStatus(self.cmd, results) 259 return results

260 d.addCallback(_gotResults) 261 d.addCallbacks(self.finished, self.checkDisconnect) 262 return d 263 264 @defer.deferredGenerator

265 - def _rmdir(self, dir):

266 cmd = buildstep.RemoteCommand('rmdir', 267 {'dir': dir, 'logEnviron': self.logEnviron }) 268 cmd.useLog(self.stdio_log, False) 269 wfd = defer.waitForDeferred( 270 self.runCommand(cmd)) 271 yield wfd 272 wfd.getResult() 273 if cmd.rc != 0: 274 raise buildstep.BuildStepFailed()

275

276 - def _dovccmd(self, command, collectStdout=False):

277 assert command, "No command specified" 278 command.extend(['--non-interactive', '--no-auth-cache']) 279 if self.username: 280 command.extend(['--username', self.username]) 281 if self.password: 282 command.extend(['--password', self.password]) 283 if self.depth: 284 command.extend(['--depth', self.depth]) 285 if self.extra_args: 286 command.extend(self.extra_args) 287 288 cmd = buildstep.RemoteShellCommand(self.workdir, ['svn'] + command, 289 env=self.env, 290 logEnviron=self.logEnviron, 291 collectStdout=collectStdout) 292 cmd.useLog(self.stdio_log, False) 293 log.msg("Starting SVN command : svn %s" % (" ".join(command), )) 294 d = self.runCommand(cmd) 295 def evaluateCommand(cmd): 296 if cmd.rc != 0: 297 log.msg("Source step failed while running command %s" % cmd) 298 raise buildstep.BuildStepFailed() 299 if collectStdout: 300 return cmd.stdout 301 else: 302 return cmd.rc

303 d.addCallback(lambda _: evaluateCommand(cmd)) 304 return d 305

306 - def getRepoUrl(self, branch):

307 ''' Compute the svn url that will be passed to the svn remote command ''' 308 if self.repourl: 309 return self.repourl 310 else: 311 if branch is None: 312 m = ("The SVN source step belonging to builder '%s' does not know " 313 "which branch to work with. This means that the change source " 314 "did not specify a branch and that defaultBranch is None." \ 315 % self.build.builder.name) 316 raise RuntimeError(m) 317 318 computed = self.baseURL 319 320 if self.branch_placeholder in self.baseURL: 321 return computed.replace(self.branch_placeholder, branch) 322 else: 323 return computed + branch

324

325 - def _getMethod(self):

326 if self.method is not None and self.mode != 'incremental': 327 return self.method 328 elif self.mode == 'incremental': 329 return None 330 elif self.method is None and self.mode == 'full': 331 return 'fresh'

332 333 @defer.deferredGenerator

334 - def _sourcedirIsUpdatable(self):

335 # first, perform a stat to ensure that this is really an svn directory 336 cmd = buildstep.RemoteCommand('stat', {'file': self.workdir + '/.svn', 337 'logEnviron': self.logEnviron,}) 338 cmd.useLog(self.stdio_log, False) 339 wfd = defer.waitForDeferred( 340 self.runCommand(cmd)) 341 yield wfd 342 wfd.getResult() 343 344 if cmd.rc != 0: 345 yield False 346 return 347 348 # then run 'svn info' to check that the URL matches our repourl 349 wfd = defer.waitForDeferred( 350 self._dovccmd(['info'], collectStdout=True)) 351 yield wfd 352 stdout = wfd.getResult() 353 354 # extract the URL, handling whitespace carefully so that \r\n works 355 # is a line terminator 356 mo = re.search('^URL:\s*(.*?)\s*$', stdout, re.M) 357 yield mo and mo.group(1) == self.repourl 358 return

359

360 - def parseGotRevision(self, _):

361 # if this was a full/export, then we need to check svnversion in the 362 # *source* directory, not the build directory 363 svnversion_dir = self.workdir 364 if self.mode == 'full' and self.method == 'export': 365 svnversion_dir = 'source' 366 367 cmd = buildstep.RemoteShellCommand(svnversion_dir, ['svnversion'], 368 env=self.env, 369 logEnviron=self.logEnviron, 370 collectStdout=True) 371 cmd.useLog(self.stdio_log, False) 372 d = self.runCommand(cmd) 373 def _setrev(_): 374 stdout = cmd.stdout.strip() 375 revision = stdout.rstrip('MSP') 376 revision = revision.split(':')[-1] 377 try: 378 int(revision) 379 except ValueError: 380 msg =("SVN.parseGotRevision unable to parse output " 381 "of svnversion: '%s'" % stdout) 382 log.msg(msg) 383 raise buildstep.BuildStepFailed() 384 385 log.msg("Got SVN revision %s" % (revision, )) 386 self.setProperty('got_revision', revision, 'Source') 387 return 0

388 d.addCallback(lambda _: _setrev(cmd.rc)) 389 return d 390

391 - def purge(self, ignore_ignores):

392 """Delete everything that shown up on status.""" 393 command = ['status', '--xml'] 394 if ignore_ignores: 395 command.append('--no-ignore') 396 d = self._dovccmd(command, collectStdout=True) 397 def parseAndRemove(stdout): 398 files = [] 399 for filename in self.getUnversionedFiles(stdout, self.keep_on_purge): 400 filename = self.workdir+'/'+str(filename) 401 files.append(filename) 402 if len(files) == 0: 403 d = defer.succeed(0) 404 else: 405 if not self.slaveVersionIsOlderThan('rmdir', '2.14'): 406 d = self.removeFiles(files) 407 else: 408 cmd = buildstep.RemoteCommand('rmdir', {'dir': files, 409 'logEnviron': 410 self.logEnviron,}) 411 cmd.useLog(self.stdio_log, False) 412 d = self.runCommand(cmd) 413 d.addCallback(lambda _: cmd.rc) 414 return d

415 d.addCallback(parseAndRemove) 416 def evaluateCommand(rc): 417 if rc != 0: 418 log.msg("Failed removing files") 419 raise buildstep.BuildStepFailed() 420 return rc 421 d.addCallback(evaluateCommand) 422 return d 423 424 @staticmethod

425 - def getUnversionedFiles(xmlStr, keep_on_purge):

426 try: 427 result_xml = xml.dom.minidom.parseString(xmlStr) 428 except xml.parsers.expat.ExpatError: 429 log.err("Corrupted xml, aborting step") 430 raise buildstep.BuildStepFailed() 431 432 for entry in result_xml.getElementsByTagName('entry'): 433 (wc_status,) = entry.getElementsByTagName('wc-status') 434 if wc_status.getAttribute('item') == 'external': 435 continue 436 if wc_status.getAttribute('item') == 'missing': 437 continue 438 filename = entry.getAttribute('path') 439 if filename in keep_on_purge or filename == '': 440 continue 441 yield filename

442 443 @defer.deferredGenerator

444 - def removeFiles(self, files):

445 for filename in files: 446 cmd = buildstep.RemoteCommand('rmdir', {'dir': filename, 447 'logEnviron': self.logEnviron,}) 448 cmd.useLog(self.stdio_log, False) 449 wfd = defer.waitForDeferred(self.runCommand(cmd)) 450 yield wfd 451 wfd.getResult() 452 if cmd.rc != 0: 453 yield cmd.rc 454 return 455 yield 0

456

457 - def checkSvn(self):

458 cmd = buildstep.RemoteShellCommand(self.workdir, ['svn', '--version'], 459 env=self.env, 460 logEnviron=self.logEnviron) 461 cmd.useLog(self.stdio_log, False) 462 d = self.runCommand(cmd) 463 def evaluate(cmd): 464 if cmd.rc != 0: 465 return False 466 return True

467 d.addCallback(lambda _: evaluate(cmd)) 468 return d 469

470 - def computeSourceRevision(self, changes):

471 if not changes or None in [c.revision for c in changes]: 472 return None 473 lastChange = max([int(c.revision) for c in changes]) 474 return lastChange

475

Source Code for Module buildbot.steps.source.svn