buildbot.steps.source.svn

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 import re 17 import xml.dom.minidom 18 import xml.parsers.expat 19 20 from twisted.python import log 21 from twisted.internet import defer 22 23 from buildbot.process import buildstep 24 from buildbot.steps.source.base import Source 25 from buildbot.interfaces import BuildSlaveTooOldError 26 from buildbot.config import ConfigErrors

27 28 -class SVN(Source):

29 """I perform Subversion checkout/update operations.""" 30 31 name = 'svn' 32 33 renderables = [ 'repourl' ] 34 possible_modes = ('incremental', 'full') 35 possible_methods = ('clean', 'fresh', 'clobber', 'copy', 'export', None) 36

37 - def __init__(self, repourl=None, mode='incremental', 38 method=None, username=None, 39 password=None, extra_args=None, keep_on_purge=None, 40 depth=None, **kwargs):

41 42 self.repourl = repourl 43 self.username = username 44 self.password = password 45 self.extra_args = extra_args 46 self.keep_on_purge = keep_on_purge or [] 47 self.depth = depth 48 self.method=method 49 self.mode = mode 50 Source.__init__(self, **kwargs) 51 errors = [] 52 if self.mode not in self.possible_modes: 53 errors.append("mode %s is not one of %s" % (self.mode, self.possible_modes)) 54 if self.method not in self.possible_methods: 55 errors.append("method %s is not one of %s" % (self.method, self.possible_methods)) 56 57 if repourl is None: 58 errors.append("you must provide repourl") 59 60 if errors: 61 raise ConfigErrors(errors)

62

63 - def startVC(self, branch, revision, patch):

64 self.revision = revision 65 self.method = self._getMethod() 66 self.stdio_log = self.addLog("stdio") 67 68 d = self.checkSvn() 69 def checkInstall(svnInstalled): 70 if not svnInstalled: 71 raise BuildSlaveTooOldError("SVN is not installed on slave") 72 return 0

73 d.addCallback(checkInstall) 74 75 if self.mode == 'full': 76 d.addCallback(self.full) 77 elif self.mode == 'incremental': 78 d.addCallback(self.incremental) 79 d.addCallback(self.parseGotRevision) 80 d.addCallback(self.finish) 81 d.addErrback(self.failed) 82 return d

83 84 @defer.inlineCallbacks

85 - def full(self, _):

86 if self.method == 'clobber': 87 yield self.clobber() 88 return 89 elif self.method in ['copy', 'export']: 90 yield self.copy() 91 return 92 93 updatable = yield self._sourcedirIsUpdatable() 94 if not updatable: 95 # blow away the old (un-updatable) directory 96 yield self._rmdir(self.workdir) 97 98 # then do a checkout 99 checkout_cmd = ['checkout', self.repourl, '.'] 100 if self.revision: 101 checkout_cmd.extend(["--revision", str(self.revision)]) 102 yield self._dovccmd(checkout_cmd) 103 elif self.method == 'clean': 104 yield self.clean() 105 elif self.method == 'fresh': 106 yield self.fresh()

107 108 @defer.inlineCallbacks

109 - def incremental(self, _):

110 updatable = yield self._sourcedirIsUpdatable() 111 112 if not updatable: 113 # blow away the old (un-updatable) directory 114 yield self._rmdir(self.workdir) 115 116 # and plan to do a checkout 117 command = ['checkout', self.repourl, '.'] 118 else: 119 # otherwise, do an update 120 command = ['update'] 121 122 if self.revision: 123 command.extend(['--revision', str(self.revision)]) 124 125 yield self._dovccmd(command)

126 127 @defer.inlineCallbacks

128 - def clobber(self):

129 cmd = buildstep.RemoteCommand('rmdir', {'dir': self.workdir, 130 'logEnviron': self.logEnviron,}) 131 cmd.useLog(self.stdio_log, False) 132 yield self.runCommand(cmd) 133 if cmd.didFail(): 134 raise buildstep.BuildStepFailed() 135 136 checkout_cmd = ['checkout', self.repourl, '.'] 137 if self.revision: 138 checkout_cmd.extend(["--revision", str(self.revision)]) 139 140 yield self._dovccmd(checkout_cmd)

141

142 - def fresh(self):

143 d = self.purge(True) 144 cmd = ['update'] 145 if self.revision: 146 cmd.extend(['--revision', str(self.revision)]) 147 d.addCallback(lambda _: self._dovccmd(cmd)) 148 return d

149

150 - def clean(self):

151 d = self.purge(False) 152 cmd = ['update'] 153 if self.revision: 154 cmd.extend(['--revision', str(self.revision)]) 155 d.addCallback(lambda _: self._dovccmd(cmd)) 156 return d

157 158 @defer.inlineCallbacks

159 - def copy(self):

160 cmd = buildstep.RemoteCommand('rmdir', {'dir': self.workdir, 161 'logEnviron': self.logEnviron,}) 162 cmd.useLog(self.stdio_log, False) 163 yield self.runCommand(cmd) 164 165 if cmd.didFail(): 166 raise buildstep.BuildStepFailed() 167 168 # temporarily set workdir = 'source' and do an incremental checkout 169 try: 170 old_workdir = self.workdir 171 self.workdir = 'source' 172 yield self.incremental(None) 173 except: # finally doesn't work in python-2.4 174 self.workdir = old_workdir 175 raise 176 self.workdir = old_workdir 177 178 # if we're copying, copy; otherwise, export from source to build 179 if self.method == 'copy': 180 cmd = buildstep.RemoteCommand('cpdir', 181 { 'fromdir': 'source', 'todir':self.workdir, 182 'logEnviron': self.logEnviron }) 183 else: 184 export_cmd = ['svn', 'export'] 185 if self.revision: 186 export_cmd.extend(["--revision", str(self.revision)]) 187 export_cmd.extend(['source', self.workdir]) 188 189 cmd = buildstep.RemoteShellCommand('', export_cmd, 190 env=self.env, logEnviron=self.logEnviron, timeout=self.timeout) 191 cmd.useLog(self.stdio_log, False) 192 193 yield self.runCommand(cmd) 194 195 if cmd.didFail(): 196 raise buildstep.BuildStepFailed()

197

198 - def finish(self, res):

199 d = defer.succeed(res) 200 def _gotResults(results): 201 self.setStatus(self.cmd, results) 202 return results

203 d.addCallback(_gotResults) 204 d.addCallbacks(self.finished, self.checkDisconnect) 205 return d 206 207 @defer.inlineCallbacks

208 - def _rmdir(self, dir):

209 cmd = buildstep.RemoteCommand('rmdir', 210 {'dir': dir, 'logEnviron': self.logEnviron }) 211 cmd.useLog(self.stdio_log, False) 212 yield self.runCommand(cmd) 213 if cmd.didFail(): 214 raise buildstep.BuildStepFailed()

215

216 - def _dovccmd(self, command, collectStdout=False):

217 assert command, "No command specified" 218 command.extend(['--non-interactive', '--no-auth-cache']) 219 if self.username: 220 command.extend(['--username', self.username]) 221 if self.password: 222 command.extend(['--password', self.password]) 223 if self.depth: 224 command.extend(['--depth', self.depth]) 225 if self.extra_args: 226 command.extend(self.extra_args) 227 228 cmd = buildstep.RemoteShellCommand(self.workdir, ['svn'] + command, 229 env=self.env, 230 logEnviron=self.logEnviron, 231 timeout=self.timeout, 232 collectStdout=collectStdout) 233 cmd.useLog(self.stdio_log, False) 234 log.msg("Starting SVN command : svn %s" % (" ".join(command), )) 235 d = self.runCommand(cmd) 236 def evaluateCommand(cmd): 237 if cmd.didFail(): 238 log.msg("Source step failed while running command %s" % cmd) 239 raise buildstep.BuildStepFailed() 240 if collectStdout: 241 return cmd.stdout 242 else: 243 return cmd.rc

244 d.addCallback(lambda _: evaluateCommand(cmd)) 245 return d 246

247 - def _getMethod(self):

248 if self.method is not None and self.mode != 'incremental': 249 return self.method 250 elif self.mode == 'incremental': 251 return None 252 elif self.method is None and self.mode == 'full': 253 return 'fresh'

254 255 @defer.inlineCallbacks

256 - def _sourcedirIsUpdatable(self):

257 # first, perform a stat to ensure that this is really an svn directory 258 cmd = buildstep.RemoteCommand('stat', {'file': self.workdir + '/.svn', 259 'logEnviron': self.logEnviron,}) 260 cmd.useLog(self.stdio_log, False) 261 yield self.runCommand(cmd) 262 263 if cmd.didFail(): 264 defer.returnValue(False) 265 return 266 267 # then run 'svn info' to check that the URL matches our repourl 268 stdout = yield self._dovccmd(['info'], collectStdout=True) 269 270 # extract the URL, handling whitespace carefully so that \r\n works 271 # is a line terminator 272 mo = re.search('^URL:\s*(.*?)\s*$', stdout, re.M) 273 defer.returnValue(mo and mo.group(1) == self.repourl) 274 return

275

276 - def parseGotRevision(self, _):

277 # if this was a full/export, then we need to check svnversion in the 278 # *source* directory, not the build directory 279 svnversion_dir = self.workdir 280 if self.mode == 'full' and self.method == 'export': 281 svnversion_dir = 'source' 282 283 cmd = buildstep.RemoteShellCommand(svnversion_dir, ['svnversion'], 284 env=self.env, 285 logEnviron=self.logEnviron, 286 timeout=self.timeout, 287 collectStdout=True) 288 cmd.useLog(self.stdio_log, False) 289 d = self.runCommand(cmd) 290 def _setrev(_): 291 stdout = cmd.stdout.strip() 292 revision = stdout.rstrip('MSP') 293 revision = revision.split(':')[-1] 294 try: 295 int(revision) 296 except ValueError: 297 msg =("SVN.parseGotRevision unable to parse output " 298 "of svnversion: '%s'" % stdout) 299 log.msg(msg) 300 raise buildstep.BuildStepFailed() 301 302 log.msg("Got SVN revision %s" % (revision, )) 303 self.updateSourceProperty('got_revision', revision) 304 return 0

305 d.addCallback(lambda _: _setrev(cmd.rc)) 306 return d 307

308 - def purge(self, ignore_ignores):

309 """Delete everything that shown up on status.""" 310 command = ['status', '--xml'] 311 if ignore_ignores: 312 command.append('--no-ignore') 313 d = self._dovccmd(command, collectStdout=True) 314 def parseAndRemove(stdout): 315 files = [] 316 for filename in self.getUnversionedFiles(stdout, self.keep_on_purge): 317 filename = self.workdir+'/'+str(filename) 318 files.append(filename) 319 if len(files) == 0: 320 d = defer.succeed(0) 321 else: 322 if self.slaveVersionIsOlderThan('rmdir', '2.14'): 323 d = self.removeFiles(files) 324 else: 325 cmd = buildstep.RemoteCommand('rmdir', {'dir': files, 326 'logEnviron': 327 self.logEnviron,}) 328 cmd.useLog(self.stdio_log, False) 329 d = self.runCommand(cmd) 330 d.addCallback(lambda _: cmd.rc) 331 return d

332 d.addCallback(parseAndRemove) 333 def evaluateCommand(rc): 334 if rc != 0: 335 log.msg("Failed removing files") 336 raise buildstep.BuildStepFailed() 337 return rc 338 d.addCallback(evaluateCommand) 339 return d 340 341 @staticmethod

342 - def getUnversionedFiles(xmlStr, keep_on_purge):

343 try: 344 result_xml = xml.dom.minidom.parseString(xmlStr) 345 except xml.parsers.expat.ExpatError: 346 log.err("Corrupted xml, aborting step") 347 raise buildstep.BuildStepFailed() 348 349 for entry in result_xml.getElementsByTagName('entry'): 350 (wc_status,) = entry.getElementsByTagName('wc-status') 351 if wc_status.getAttribute('item') == 'external': 352 continue 353 if wc_status.getAttribute('item') == 'missing': 354 continue 355 filename = entry.getAttribute('path') 356 if filename in keep_on_purge or filename == '': 357 continue 358 yield filename

359 360 @defer.inlineCallbacks

361 - def removeFiles(self, files):

362 for filename in files: 363 cmd = buildstep.RemoteCommand('rmdir', {'dir': filename, 364 'logEnviron': self.logEnviron,}) 365 cmd.useLog(self.stdio_log, False) 366 yield self.runCommand(cmd) 367 if cmd.rc != 0: 368 defer.returnValue(cmd.rc) 369 return 370 defer.returnValue(0)

371

372 - def checkSvn(self):

373 cmd = buildstep.RemoteShellCommand(self.workdir, ['svn', '--version'], 374 env=self.env, 375 logEnviron=self.logEnviron, 376 timeout=self.timeout) 377 cmd.useLog(self.stdio_log, False) 378 d = self.runCommand(cmd) 379 def evaluate(cmd): 380 if cmd.rc != 0: 381 return False 382 return True

383 d.addCallback(lambda _: evaluate(cmd)) 384 return d 385

386 - def computeSourceRevision(self, changes):

387 if not changes or None in [c.revision for c in changes]: 388 return None 389 lastChange = max([int(c.revision) for c in changes]) 390 return lastChange

391

Source Code for Module buildbot.steps.source.svn