buildbot.slave.commands.vcs

1 import os, re, sys, shutil, time 2 3 from xml.dom.minidom import parseString 4 5 from twisted.python import log, failure, runtime 6 from twisted.internet import defer, reactor 7 8 from buildbot.slave.commands.base import Command, ShellCommand, AbandonChain, command_version, Obfuscated 9 from buildbot.slave.commands.registry import registerSlaveCommand 10 from buildbot.slave.commands.utils import getCommand, rmdirRecursive 11 from buildbot.util import remove_userpassword 12

13 -class SourceBase(Command):

14 """Abstract base class for Version Control System operations (checkout 15 and update). This class extracts the following arguments from the 16 dictionary received from the master: 17 18 - ['workdir']: (required) the subdirectory where the buildable sources 19 should be placed 20 21 - ['mode']: one of update/copy/clobber/export, defaults to 'update' 22 23 - ['revision']: If not None, this is an int or string which indicates 24 which sources (along a time-like axis) should be used. 25 It is the thing you provide as the CVS -r or -D 26 argument. 27 28 - ['patch']: If not None, this is a tuple of (striplevel, patch) 29 which contains a patch that should be applied after the 30 checkout has occurred. Once applied, the tree is no 31 longer eligible for use with mode='update', and it only 32 makes sense to use this in conjunction with a 33 ['revision'] argument. striplevel is an int, and patch 34 is a string in standard unified diff format. The patch 35 will be applied with 'patch -p%d <PATCH', with 36 STRIPLEVEL substituted as %d. The command will fail if 37 the patch process fails (rejected hunks). 38 39 - ['timeout']: seconds of silence tolerated before we kill off the 40 command 41 42 - ['maxTime']: seconds before we kill off the command 43 44 - ['retry']: If not None, this is a tuple of (delay, repeats) 45 which means that any failed VC updates should be 46 reattempted, up to REPEATS times, after a delay of 47 DELAY seconds. This is intended to deal with slaves 48 that experience transient network failures. 49 """ 50 51 sourcedata = "" 52

53 - def setup(self, args):

54 # if we need to parse the output, use this environment. Otherwise 55 # command output will be in whatever the buildslave's native language 56 # has been set to. 57 self.env = os.environ.copy() 58 self.env['LC_MESSAGES'] = "C" 59 60 self.workdir = args['workdir'] 61 self.mode = args.get('mode', "update") 62 self.revision = args.get('revision') 63 self.patch = args.get('patch') 64 self.timeout = args.get('timeout', 120) 65 self.maxTime = args.get('maxTime', None) 66 self.retry = args.get('retry')

67 # VC-specific subclasses should override this to extract more args. 68 # Make sure to upcall! 69

70 - def start(self):

71 self.sendStatus({'header': "starting " + self.header + "\n"}) 72 self.command = None 73 74 # self.srcdir is where the VC system should put the sources 75 if self.mode == "copy": 76 self.srcdir = "source" # hardwired directory name, sorry 77 else: 78 self.srcdir = self.workdir 79 self.sourcedatafile = os.path.join(self.builder.basedir, 80 self.srcdir, 81 ".buildbot-sourcedata") 82 83 d = defer.succeed(None) 84 self.maybeClobber(d) 85 if not (self.sourcedirIsUpdateable() and self.sourcedataMatches()): 86 # the directory cannot be updated, so we have to clobber it. 87 # Perhaps the master just changed modes from 'export' to 88 # 'update'. 89 d.addCallback(self.doClobber, self.srcdir) 90 91 d.addCallback(self.doVC) 92 93 if self.mode == "copy": 94 d.addCallback(self.doCopy) 95 if self.patch: 96 d.addCallback(self.doPatch) 97 d.addCallbacks(self._sendRC, self._checkAbandoned) 98 return d

99

100 - def maybeClobber(self, d):

101 # do we need to clobber anything? 102 if self.mode in ("copy", "clobber", "export"): 103 d.addCallback(self.doClobber, self.workdir)

104

105 - def interrupt(self):

106 self.interrupted = True 107 if self.command: 108 self.command.kill("command interrupted")

109

110 - def doVC(self, res):

111 if self.interrupted: 112 raise AbandonChain(1) 113 if self.sourcedirIsUpdateable() and self.sourcedataMatches(): 114 d = self.doVCUpdate() 115 d.addCallback(self.maybeDoVCFallback) 116 else: 117 d = self.doVCFull() 118 d.addBoth(self.maybeDoVCRetry) 119 d.addCallback(self._abandonOnFailure) 120 d.addCallback(self._handleGotRevision) 121 d.addCallback(self.writeSourcedata) 122 return d

123

124 - def sourcedataMatches(self):

125 try: 126 olddata = self.readSourcedata() 127 if olddata != self.sourcedata: 128 return False 129 except IOError: 130 return False 131 return True

132

133 - def sourcedirIsPatched(self):

134 return os.path.exists(os.path.join(self.builder.basedir, 135 self.workdir, 136 ".buildbot-patched"))

137

138 - def _handleGotRevision(self, res):

139 d = defer.maybeDeferred(self.parseGotRevision) 140 d.addCallback(lambda got_revision: 141 self.sendStatus({'got_revision': got_revision})) 142 return d

143

144 - def parseGotRevision(self):

145 """Override this in a subclass. It should return a string that 146 represents which revision was actually checked out, or a Deferred 147 that will fire with such a string. If, in a future build, you were to 148 pass this 'got_revision' string in as the 'revision' component of a 149 SourceStamp, you should wind up with the same source code as this 150 checkout just obtained. 151 152 It is probably most useful to scan self.command.stdout for a string 153 of some sort. Be sure to set keepStdout=True on the VC command that 154 you run, so that you'll have something available to look at. 155 156 If this information is unavailable, just return None.""" 157 158 return None

159

160 - def readSourcedata(self):

161 return open(self.sourcedatafile, "r").read()

162

163 - def writeSourcedata(self, res):

164 open(self.sourcedatafile, "w").write(self.sourcedata) 165 return res

166

167 - def sourcedirIsUpdateable(self):

168 """Returns True if the tree can be updated.""" 169 raise NotImplementedError("this must be implemented in a subclass")

170

171 - def doVCUpdate(self):

172 """Returns a deferred with the steps to update a checkout.""" 173 raise NotImplementedError("this must be implemented in a subclass")

174

175 - def doVCFull(self):

176 """Returns a deferred with the steps to do a fresh checkout.""" 177 raise NotImplementedError("this must be implemented in a subclass")

178

179 - def maybeDoVCFallback(self, rc):

180 if type(rc) is int and rc == 0: 181 return rc 182 if self.interrupted: 183 raise AbandonChain(1) 184 msg = "update failed, clobbering and trying again" 185 self.sendStatus({'header': msg + "\n"}) 186 log.msg(msg) 187 d = self.doClobber(None, self.srcdir) 188 d.addCallback(self.doVCFallback2) 189 return d

190

191 - def doVCFallback2(self, res):

192 msg = "now retrying VC operation" 193 self.sendStatus({'header': msg + "\n"}) 194 log.msg(msg) 195 d = self.doVCFull() 196 d.addBoth(self.maybeDoVCRetry) 197 d.addCallback(self._abandonOnFailure) 198 return d

199

200 - def maybeDoVCRetry(self, res):

201 """We get here somewhere after a VC chain has finished. res could 202 be:: 203 204 - 0: the operation was successful 205 - nonzero: the operation failed. retry if possible 206 - AbandonChain: the operation failed, someone else noticed. retry. 207 - Failure: some other exception, re-raise 208 """ 209 210 if isinstance(res, failure.Failure): 211 if self.interrupted: 212 return res # don't re-try interrupted builds 213 res.trap(AbandonChain) 214 else: 215 if type(res) is int and res == 0: 216 return res 217 if self.interrupted: 218 raise AbandonChain(1) 219 # if we get here, we should retry, if possible 220 if self.retry: 221 delay, repeats = self.retry 222 if repeats >= 0: 223 self.retry = (delay, repeats-1) 224 msg = ("update failed, trying %d more times after %d seconds" 225 % (repeats, delay)) 226 self.sendStatus({'header': msg + "\n"}) 227 log.msg(msg) 228 d = defer.Deferred() 229 # we are going to do a full checkout, so a clobber is 230 # required first 231 self.doClobber(d, self.workdir) 232 d.addCallback(lambda res: self.doVCFull()) 233 d.addBoth(self.maybeDoVCRetry) 234 self._reactor.callLater(delay, d.callback, None) 235 return d 236 return res

237

238 - def doClobber(self, dummy, dirname, chmodDone=False):

239 # TODO: remove the old tree in the background 240 ## workdir = os.path.join(self.builder.basedir, self.workdir) 241 ## deaddir = self.workdir + ".deleting" 242 ## if os.path.isdir(workdir): 243 ## try: 244 ## os.rename(workdir, deaddir) 245 ## # might fail if deaddir already exists: previous deletion 246 ## # hasn't finished yet 247 ## # start the deletion in the background 248 ## # TODO: there was a solaris/NetApp/NFS problem where a 249 ## # process that was still running out of the directory we're 250 ## # trying to delete could prevent the rm-rf from working. I 251 ## # think it stalled the rm, but maybe it just died with 252 ## # permission issues. Try to detect this. 253 ## os.commands("rm -rf %s &" % deaddir) 254 ## except: 255 ## # fall back to sequential delete-then-checkout 256 ## pass 257 d = os.path.join(self.builder.basedir, dirname) 258 if runtime.platformType != "posix": 259 # if we're running on w32, use rmtree instead. It will block, 260 # but hopefully it won't take too long. 261 rmdirRecursive(d) 262 return defer.succeed(0) 263 command = ["rm", "-rf", d] 264 c = ShellCommand(self.builder, command, self.builder.basedir, 265 sendRC=0, timeout=self.timeout, maxTime=self.maxTime, 266 usePTY=False) 267 268 self.command = c 269 # sendRC=0 means the rm command will send stdout/stderr to the 270 # master, but not the rc=0 when it finishes. That job is left to 271 # _sendRC 272 d = c.start() 273 # The rm -rf may fail if there is a left-over subdir with chmod 000 274 # permissions. So if we get a failure, we attempt to chmod suitable 275 # permissions and re-try the rm -rf. 276 if chmodDone: 277 d.addCallback(self._abandonOnFailure) 278 else: 279 d.addCallback(lambda rc: self.doClobberTryChmodIfFail(rc, dirname)) 280 return d

281

282 - def doClobberTryChmodIfFail(self, rc, dirname):

283 assert isinstance(rc, int) 284 if rc == 0: 285 return defer.succeed(0) 286 # Attempt a recursive chmod and re-try the rm -rf after. 287 288 command = ["chmod", "-Rf", "u+rwx", os.path.join(self.builder.basedir, dirname)] 289 if sys.platform.startswith('freebsd'): 290 # Work around a broken 'chmod -R' on FreeBSD (it tries to recurse into a 291 # directory for which it doesn't have permission, before changing that 292 # permission) by running 'find' instead 293 command = ["find", os.path.join(self.builder.basedir, dirname), 294 '-exec', 'chmod', 'u+rwx', '{}', ';' ] 295 c = ShellCommand(self.builder, command, self.builder.basedir, 296 sendRC=0, timeout=self.timeout, maxTime=self.maxTime, 297 usePTY=False) 298 299 self.command = c 300 d = c.start() 301 d.addCallback(self._abandonOnFailure) 302 d.addCallback(lambda dummy: self.doClobber(dummy, dirname, True)) 303 return d

304

305 - def doCopy(self, res):

306 # now copy tree to workdir 307 fromdir = os.path.join(self.builder.basedir, self.srcdir) 308 todir = os.path.join(self.builder.basedir, self.workdir) 309 if runtime.platformType != "posix": 310 self.sendStatus({'header': "Since we're on a non-POSIX platform, " 311 "we're not going to try to execute cp in a subprocess, but instead " 312 "use shutil.copytree(), which will block until it is complete. " 313 "fromdir: %s, todir: %s\n" % (fromdir, todir)}) 314 shutil.copytree(fromdir, todir) 315 return defer.succeed(0) 316 317 if not os.path.exists(os.path.dirname(todir)): 318 os.makedirs(os.path.dirname(todir)) 319 if os.path.exists(todir): 320 # I don't think this happens, but just in case.. 321 log.msg("cp target '%s' already exists -- cp will not do what you think!" % todir) 322 323 command = ['cp', '-R', '-P', '-p', fromdir, todir] 324 c = ShellCommand(self.builder, command, self.builder.basedir, 325 sendRC=False, timeout=self.timeout, maxTime=self.maxTime, 326 usePTY=False) 327 self.command = c 328 d = c.start() 329 d.addCallback(self._abandonOnFailure) 330 return d

331

332 - def doPatch(self, res):

333 patchlevel = self.patch[0] 334 diff = self.patch[1] 335 root = None 336 if len(self.patch) >= 3: 337 root = self.patch[2] 338 command = [ 339 getCommand("patch"), 340 '-p%d' % patchlevel, 341 '--remove-empty-files', 342 '--force', 343 '--forward', 344 ] 345 dir = os.path.join(self.builder.basedir, self.workdir) 346 # Mark the directory so we don't try to update it later, or at least try 347 # to revert first. 348 marker = open(os.path.join(dir, ".buildbot-patched"), "w") 349 marker.write("patched\n") 350 marker.close() 351 352 # Update 'dir' with the 'root' option. Make sure it is a subdirectory 353 # of dir. 354 if (root and 355 os.path.abspath(os.path.join(dir, root) 356 ).startswith(os.path.abspath(dir))): 357 dir = os.path.join(dir, root) 358 359 # now apply the patch 360 c = ShellCommand(self.builder, command, dir, 361 sendRC=False, timeout=self.timeout, 362 maxTime=self.maxTime, initialStdin=diff, usePTY=False) 363 self.command = c 364 d = c.start() 365 d.addCallback(self._abandonOnFailure) 366 return d

367 368 369

370 -class BK(SourceBase):

371 """BitKeeper-specific VC operation. In addition to the arguments 372 handled by SourceBase, this command reads the following keys: 373 374 ['bkurl'] (required): the BK repository string 375 """ 376 377 header = "bk operation" 378

379 - def setup(self, args):

380 SourceBase.setup(self, args) 381 self.vcexe = getCommand("bk") 382 self.bkurl = args['bkurl'] 383 self.sourcedata = '"%s\n"' % self.bkurl 384 385 self.bk_args = [] 386 if args.get('extra_args', None) is not None: 387 self.bk_args.extend(args['extra_args'])

388

389 - def sourcedirIsUpdateable(self):

390 if os.path.exists(os.path.join(self.builder.basedir, 391 self.srcdir, ".buildbot-patched")): 392 return False 393 return os.path.isfile(os.path.join(self.builder.basedir, 394 self.srcdir, "BK/parent"))

395

396 - def doVCUpdate(self):

397 revision = self.args['revision'] or 'HEAD' 398 # update: possible for mode in ('copy', 'update') 399 d = os.path.join(self.builder.basedir, self.srcdir) 400 401 # Revision is ignored since the BK free client doesn't support it. 402 command = [self.vcexe, 'pull'] 403 c = ShellCommand(self.builder, command, d, 404 sendRC=False, timeout=self.timeout, 405 keepStdout=True, usePTY=False) 406 self.command = c 407 return c.start()

408

409 - def doVCFull(self):

410 411 revision_arg = '' 412 if self.args['revision']: 413 revision_arg = "-r%s" % self.args['revision'] 414 415 d = self.builder.basedir 416 417 command = [self.vcexe, 'clone', revision_arg] + self.bk_args + \ 418 [self.bkurl, self.srcdir] 419 c = ShellCommand(self.builder, command, d, 420 sendRC=False, timeout=self.timeout, 421 keepStdout=True, usePTY=False) 422 self.command = c 423 return c.start()

424

425 - def getBKVersionCommand(self):

426 """ 427 Get the (shell) command used to determine BK revision number 428 of checked-out code 429 430 return: list of strings, passable as the command argument to ShellCommand 431 """ 432 return [self.vcexe, "changes", "-r+", "-d:REV:"]

433

434 - def parseGotRevision(self):

435 c = ShellCommand(self.builder, 436 self.getBKVersionCommand(), 437 os.path.join(self.builder.basedir, self.srcdir), 438 environ=self.env, 439 sendStdout=False, sendStderr=False, sendRC=False, 440 keepStdout=True, usePTY=False) 441 d = c.start() 442 def _parse(res): 443 r_raw = c.stdout.strip() 444 got_version = None 445 try: 446 r = r_raw 447 except: 448 msg = ("BK.parseGotRevision unable to parse output: (%s)" % r_raw) 449 log.msg(msg) 450 self.sendStatus({'header': msg + "\n"}) 451 raise ValueError(msg) 452 return r

453 d.addCallback(_parse) 454 return d

455 456 registerSlaveCommand("bk", BK, command_version) 457 458 459 460

461 -class CVS(SourceBase):

462 """CVS-specific VC operation. In addition to the arguments handled by 463 SourceBase, this command reads the following keys: 464 465 ['cvsroot'] (required): the CVSROOT repository string 466 ['cvsmodule'] (required): the module to be retrieved 467 ['branch']: a '-r' tag or branch name to use for the checkout/update 468 ['login']: a string for use as a password to 'cvs login' 469 ['global_options']: a list of strings to use before the CVS verb 470 ['checkout_options']: a list of strings to use after checkout, 471 but before revision and branch specifiers 472 ['checkout_options']: a list of strings to use after export, 473 but before revision and branch specifiers 474 ['extra_options']: a list of strings to use after export and checkout, 475 but before revision and branch specifiers 476 """ 477 478 header = "cvs operation" 479

480 - def setup(self, args):

481 SourceBase.setup(self, args) 482 self.vcexe = getCommand("cvs") 483 self.cvsroot = args['cvsroot'] 484 self.cvsmodule = args['cvsmodule'] 485 self.global_options = args.get('global_options', []) 486 self.checkout_options = args.get('checkout_options', []) 487 self.export_options = args.get('export_options', []) 488 self.extra_options = args.get('extra_options', []) 489 self.branch = args.get('branch') 490 self.login = args.get('login') 491 self.sourcedata = "%s\n%s\n%s\n" % (self.cvsroot, self.cvsmodule, 492 self.branch)

493

494 - def sourcedirIsUpdateable(self):

495 return (not self.sourcedirIsPatched() and 496 os.path.isdir(os.path.join(self.builder.basedir, 497 self.srcdir, "CVS")))

498

499 - def start(self):

500 if self.login is not None: 501 # need to do a 'cvs login' command first 502 d = self.builder.basedir 503 command = ([self.vcexe, '-d', self.cvsroot] + self.global_options 504 + ['login']) 505 c = ShellCommand(self.builder, command, d, 506 sendRC=False, timeout=self.timeout, 507 maxTime=self.maxTime, 508 initialStdin=self.login+"\n", usePTY=False) 509 self.command = c 510 d = c.start() 511 d.addCallback(self._abandonOnFailure) 512 d.addCallback(self._didLogin) 513 return d 514 else: 515 return self._didLogin(None)

516

517 - def _didLogin(self, res):

518 # now we really start 519 return SourceBase.start(self)

520

521 - def doVCUpdate(self):

522 d = os.path.join(self.builder.basedir, self.srcdir) 523 command = [self.vcexe, '-z3'] + self.global_options + ['update', '-dP'] 524 if self.branch: 525 command += ['-r', self.branch] 526 if self.revision: 527 command += ['-D', self.revision] 528 c = ShellCommand(self.builder, command, d, 529 sendRC=False, timeout=self.timeout, 530 maxTime=self.maxTime, usePTY=False) 531 self.command = c 532 return c.start()

533

534 - def doVCFull(self):

535 d = self.builder.basedir 536 if self.mode == "export": 537 verb = "export" 538 else: 539 verb = "checkout" 540 command = ([self.vcexe, '-d', self.cvsroot, '-z3'] + 541 self.global_options + 542 [verb, '-d', self.srcdir]) 543 544 if verb == "checkout": 545 command += self.checkout_options 546 else: 547 command += self.export_options 548 command += self.extra_options 549 550 if self.branch: 551 command += ['-r', self.branch] 552 if self.revision: 553 command += ['-D', self.revision] 554 command += [self.cvsmodule] 555 556 c = ShellCommand(self.builder, command, d, 557 sendRC=False, timeout=self.timeout, 558 maxTime=self.maxTime, usePTY=False) 559 self.command = c 560 return c.start()

561

562 - def parseGotRevision(self):

563 # CVS does not have any kind of revision stamp to speak of. We return 564 # the current timestamp as a best-effort guess, but this depends upon 565 # the local system having a clock that is 566 # reasonably-well-synchronized with the repository. 567 return time.strftime("%Y-%m-%d %H:%M:%S +0000", time.gmtime())

568 569 registerSlaveCommand("cvs", CVS, command_version) 570

571 -class SVN(SourceBase):

572 """Subversion-specific VC operation. In addition to the arguments 573 handled by SourceBase, this command reads the following keys: 574 575 ['svnurl'] (required): the SVN repository string 576 ['username']: Username passed to the svn command 577 ['password']: Password passed to the svn command 578 ['keep_on_purge']: Files and directories to keep between updates 579 ['ignore_ignores']: Ignore ignores when purging changes 580 ['always_purge']: Always purge local changes after each build 581 ['depth']: Pass depth argument to subversion 1.5+ 582 """ 583 584 header = "svn operation" 585

586 - def setup(self, args):

587 SourceBase.setup(self, args) 588 self.vcexe = getCommand("svn") 589 self.svnurl = args['svnurl'] 590 self.sourcedata = "%s\n" % self.svnurl 591 self.keep_on_purge = args.get('keep_on_purge', []) 592 self.keep_on_purge.append(".buildbot-sourcedata") 593 self.ignore_ignores = args.get('ignore_ignores', True) 594 self.always_purge = args.get('always_purge', False) 595 596 self.svn_args = [] 597 if args.has_key('username'): 598 self.svn_args.extend(["--username", args['username']]) 599 if args.has_key('password'): 600 self.svn_args.extend(["--password", Obfuscated(args['password'], "XXXX")]) 601 if args.get('extra_args', None) is not None: 602 self.svn_args.extend(args['extra_args']) 603 604 if args.has_key('depth'): 605 self.svn_args.extend(["--depth",args['depth']])

606

607 - def _dovccmd(self, command, args, rootdir=None, cb=None, **kwargs):

608 if rootdir is None: 609 rootdir = os.path.join(self.builder.basedir, self.srcdir) 610 fullCmd = [self.vcexe, command, '--non-interactive', '--no-auth-cache'] 611 fullCmd.extend(self.svn_args) 612 fullCmd.extend(args) 613 c = ShellCommand(self.builder, fullCmd, rootdir, 614 environ=self.env, sendRC=False, timeout=self.timeout, 615 maxTime=self.maxTime, usePTY=False, **kwargs) 616 self.command = c 617 d = c.start() 618 if cb: 619 d.addCallback(self._abandonOnFailure) 620 d.addCallback(cb) 621 return d

622

623 - def sourcedirIsUpdateable(self):

624 return os.path.isdir(os.path.join(self.builder.basedir, 625 self.srcdir, ".svn"))

626

627 - def doVCUpdate(self):

628 if self.sourcedirIsPatched() or self.always_purge: 629 return self._purgeAndUpdate() 630 revision = self.args['revision'] or 'HEAD' 631 # update: possible for mode in ('copy', 'update') 632 return self._dovccmd('update', ['--revision', str(revision)], 633 keepStdout=True)

634

635 - def doVCFull(self):

636 revision = self.args['revision'] or 'HEAD' 637 args = ['--revision', str(revision), self.svnurl, self.srcdir] 638 if self.mode == "export": 639 command = 'export' 640 else: 641 # mode=='clobber', or copy/update on a broken workspace 642 command = 'checkout' 643 return self._dovccmd(command, args, rootdir=self.builder.basedir, 644 keepStdout=True)

645

646 - def _purgeAndUpdate(self):

647 """svn revert has several corner cases that make it unpractical. 648 649 Use the Force instead and delete everything that shows up in status.""" 650 args = ['--xml'] 651 if self.ignore_ignores: 652 args.append('--no-ignore') 653 return self._dovccmd('status', args, keepStdout=True, sendStdout=False, 654 cb=self._purgeAndUpdate2)

655

656 - def _purgeAndUpdate2(self, res):

657 """Delete everything that shown up on status.""" 658 result_xml = parseString(self.command.stdout) 659 for entry in result_xml.getElementsByTagName('entry'): 660 filename = entry.getAttribute('path') 661 if filename in self.keep_on_purge: 662 continue 663 filepath = os.path.join(self.builder.basedir, self.workdir, 664 filename) 665 self.sendStatus({'stdout': "%s\n" % filepath}) 666 if os.path.isfile(filepath): 667 os.chmod(filepath, 0700) 668 os.remove(filepath) 669 else: 670 rmdirRecursive(filepath) 671 # Now safe to update. 672 revision = self.args['revision'] or 'HEAD' 673 return self._dovccmd('update', ['--revision', str(revision)], 674 keepStdout=True)

675

676 - def getSvnVersionCommand(self):

677 """ 678 Get the (shell) command used to determine SVN revision number 679 of checked-out code 680 681 return: list of strings, passable as the command argument to ShellCommand 682 """ 683 # svn checkout operations finish with 'Checked out revision 16657.' 684 # svn update operations finish the line 'At revision 16654.' 685 # But we don't use those. Instead, run 'svnversion'. 686 svnversion_command = getCommand("svnversion") 687 # older versions of 'svnversion' (1.1.4) require the WC_PATH 688 # argument, newer ones (1.3.1) do not. 689 return [svnversion_command, "."]

690

691 - def parseGotRevision(self):

692 c = ShellCommand(self.builder, 693 self.getSvnVersionCommand(), 694 os.path.join(self.builder.basedir, self.srcdir), 695 environ=self.env, 696 sendStdout=False, sendStderr=False, sendRC=False, 697 keepStdout=True, usePTY=False) 698 d = c.start() 699 def _parse(res): 700 r_raw = c.stdout.strip() 701 # Extract revision from the version "number" string 702 r = r_raw.rstrip('MS') 703 r = r.split(':')[-1] 704 got_version = None 705 try: 706 got_version = int(r) 707 except ValueError: 708 msg =("SVN.parseGotRevision unable to parse output " 709 "of svnversion: '%s'" % r_raw) 710 log.msg(msg) 711 self.sendStatus({'header': msg + "\n"}) 712 return got_version

713 d.addCallback(_parse) 714 return d

715 716 717 registerSlaveCommand("svn", SVN, command_version) 718

719 -class Darcs(SourceBase):

720 """Darcs-specific VC operation. In addition to the arguments 721 handled by SourceBase, this command reads the following keys: 722 723 ['repourl'] (required): the Darcs repository string 724 """ 725 726 header = "darcs operation" 727

728 - def setup(self, args):

729 SourceBase.setup(self, args) 730 self.vcexe = getCommand("darcs") 731 self.repourl = args['repourl'] 732 self.sourcedata = "%s\n" % self.repourl 733 self.revision = self.args.get('revision')

734

735 - def sourcedirIsUpdateable(self):

736 # checking out a specific revision requires a full 'darcs get' 737 return (not self.revision and 738 not self.sourcedirIsPatched() and 739 os.path.isdir(os.path.join(self.builder.basedir, 740 self.srcdir, "_darcs")))

741

742 - def doVCUpdate(self):

743 assert not self.revision 744 # update: possible for mode in ('copy', 'update') 745 d = os.path.join(self.builder.basedir, self.srcdir) 746 command = [self.vcexe, 'pull', '--all', '--verbose'] 747 c = ShellCommand(self.builder, command, d, 748 sendRC=False, timeout=self.timeout, 749 maxTime=self.maxTime, usePTY=False) 750 self.command = c 751 return c.start()

752

753 - def doVCFull(self):

754 # checkout or export 755 d = self.builder.basedir 756 command = [self.vcexe, 'get', '--verbose', '--partial', 757 '--repo-name', self.srcdir] 758 if self.revision: 759 # write the context to a file 760 n = os.path.join(self.builder.basedir, ".darcs-context") 761 f = open(n, "wb") 762 f.write(self.revision) 763 f.close() 764 # tell Darcs to use that context 765 command.append('--context') 766 command.append(n) 767 command.append(self.repourl) 768 769 c = ShellCommand(self.builder, command, d, 770 sendRC=False, timeout=self.timeout, 771 maxTime=self.maxTime, usePTY=False) 772 self.command = c 773 d = c.start() 774 if self.revision: 775 d.addCallback(self.removeContextFile, n) 776 return d

777

778 - def removeContextFile(self, res, n):

779 os.unlink(n) 780 return res

781

782 - def parseGotRevision(self):

783 # we use 'darcs context' to find out what we wound up with 784 command = [self.vcexe, "changes", "--context"] 785 c = ShellCommand(self.builder, command, 786 os.path.join(self.builder.basedir, self.srcdir), 787 environ=self.env, 788 sendStdout=False, sendStderr=False, sendRC=False, 789 keepStdout=True, usePTY=False) 790 d = c.start() 791 d.addCallback(lambda res: c.stdout) 792 return d

793 794 registerSlaveCommand("darcs", Darcs, command_version) 795

796 -class Monotone(SourceBase):

797 """Monotone-specific VC operation. In addition to the arguments handled 798 by SourceBase, this command reads the following keys: 799 800 ['server_addr'] (required): the address of the server to pull from 801 ['branch'] (required): the branch the revision is on 802 ['db_path'] (required): the local database path to use 803 ['revision'] (required): the revision to check out 804 ['monotone']: (required): path to monotone executable 805 """ 806 807 header = "monotone operation" 808

809 - def setup(self, args):

810 SourceBase.setup(self, args) 811 self.server_addr = args["server_addr"] 812 self.branch = args["branch"] 813 self.db_path = args["db_path"] 814 self.revision = args["revision"] 815 self.monotone = args["monotone"] 816 self._made_fulls = False 817 self._pull_timeout = args["timeout"]

818

819 - def _makefulls(self):

820 if not self._made_fulls: 821 basedir = self.builder.basedir 822 self.full_db_path = os.path.join(basedir, self.db_path) 823 self.full_srcdir = os.path.join(basedir, self.srcdir) 824 self._made_fulls = True

825

826 - def sourcedirIsUpdateable(self):

827 self._makefulls() 828 return (not self.sourcedirIsPatched() and 829 os.path.isfile(self.full_db_path) and 830 os.path.isdir(os.path.join(self.full_srcdir, "MT")))

831

832 - def doVCUpdate(self):

833 return self._withFreshDb(self._doUpdate)

834

835 - def _doUpdate(self):

836 # update: possible for mode in ('copy', 'update') 837 command = [self.monotone, "update", 838 "-r", self.revision, 839 "-b", self.branch] 840 c = ShellCommand(self.builder, command, self.full_srcdir, 841 sendRC=False, timeout=self.timeout, 842 maxTime=self.maxTime, usePTY=False) 843 self.command = c 844 return c.start()

845

846 - def doVCFull(self):

847 return self._withFreshDb(self._doFull)

848

849 - def _doFull(self):

850 command = [self.monotone, "--db=" + self.full_db_path, 851 "checkout", 852 "-r", self.revision, 853 "-b", self.branch, 854 self.full_srcdir] 855 c = ShellCommand(self.builder, command, self.builder.basedir, 856 sendRC=False, timeout=self.timeout, 857 maxTime=self.maxTime, usePTY=False) 858 self.command = c 859 return c.start()

860

861 - def _withFreshDb(self, callback):

862 self._makefulls() 863 # first ensure the db exists and is usable 864 if os.path.isfile(self.full_db_path): 865 # already exists, so run 'db migrate' in case monotone has been 866 # upgraded under us 867 command = [self.monotone, "db", "migrate", 868 "--db=" + self.full_db_path] 869 else: 870 # We'll be doing an initial pull, so up the timeout to 3 hours to 871 # make sure it will have time to complete. 872 self._pull_timeout = max(self._pull_timeout, 3 * 60 * 60) 873 self.sendStatus({"header": "creating database %s\n" 874 % (self.full_db_path,)}) 875 command = [self.monotone, "db", "init", 876 "--db=" + self.full_db_path] 877 c = ShellCommand(self.builder, command, self.builder.basedir, 878 sendRC=False, timeout=self.timeout, 879 maxTime=self.maxTime, usePTY=False) 880 self.command = c 881 d = c.start() 882 d.addCallback(self._abandonOnFailure) 883 d.addCallback(self._didDbInit) 884 d.addCallback(self._didPull, callback) 885 return d

886

887 - def _didDbInit(self, res):

888 command = [self.monotone, "--db=" + self.full_db_path, 889 "pull", "--ticker=dot", self.server_addr, self.branch] 890 c = ShellCommand(self.builder, command, self.builder.basedir, 891 sendRC=False, timeout=self._pull_timeout, 892 maxTime=self.maxTime, usePTY=False) 893 self.sendStatus({"header": "pulling %s from %s\n" 894 % (self.branch, self.server_addr)}) 895 self.command = c 896 return c.start()

897

898 - def _didPull(self, res, callback):

899 return callback()

900 901 registerSlaveCommand("monotone", Monotone, command_version) 902 903

904 -class Git(SourceBase):

905 """Git specific VC operation. In addition to the arguments 906 handled by SourceBase, this command reads the following keys: 907 908 ['repourl'] (required): the upstream GIT repository string 909 ['branch'] (optional): which version (i.e. branch or tag) to 910 retrieve. Default: "master". 911 ['submodules'] (optional): whether to initialize and update 912 submodules. Default: False. 913 ['ignore_ignores']: ignore ignores when purging changes. 914 """ 915 916 header = "git operation" 917

918 - def setup(self, args):

919 SourceBase.setup(self, args) 920 self.vcexe = getCommand("git") 921 self.repourl = args['repourl'] 922 self.branch = args.get('branch') 923 if not self.branch: 924 self.branch = "master" 925 self.sourcedata = "%s %s\n" % (self.repourl, self.branch) 926 self.submodules = args.get('submodules') 927 self.ignore_ignores = args.get('ignore_ignores', True)

928

929 - def _fullSrcdir(self):

930 return os.path.join(self.builder.basedir, self.srcdir)

931

932 - def _commitSpec(self):

933 if self.revision: 934 return self.revision 935 return self.branch

936

937 - def sourcedirIsUpdateable(self):

938 return os.path.isdir(os.path.join(self._fullSrcdir(), ".git"))

939

940 - def _dovccmd(self, command, cb=None, **kwargs):

941 c = ShellCommand(self.builder, [self.vcexe] + command, self._fullSrcdir(), 942 sendRC=False, timeout=self.timeout, 943 maxTime=self.maxTime, usePTY=False, **kwargs) 944 self.command = c 945 d = c.start() 946 if cb: 947 d.addCallback(self._abandonOnFailure) 948 d.addCallback(cb) 949 return d

950 951 # If the repourl matches the sourcedata file, then 952 # we can say that the sourcedata matches. We can 953 # ignore branch changes, since Git can work with 954 # many branches fetched, and we deal with it properly 955 # in doVCUpdate.

956 - def sourcedataMatches(self):

957 try: 958 olddata = self.readSourcedata() 959 if not olddata.startswith(self.repourl+' '): 960 return False 961 except IOError: 962 return False 963 return True

964

965 - def _cleanSubmodules(self, res):

966 command = ['submodule', 'foreach', 'git', 'clean', '-d', '-f'] 967 if self.ignore_ignores: 968 command.append('-x') 969 return self._dovccmd(command)

970

971 - def _updateSubmodules(self, res):

972 return self._dovccmd(['submodule', 'update'], self._cleanSubmodules)

973

974 - def _initSubmodules(self, res):

975 if self.submodules: 976 return self._dovccmd(['submodule', 'init'], self._updateSubmodules) 977 else: 978 return defer.succeed(0)

979

980 - def _didHeadCheckout(self, res):

981 # Rename branch, so that the repo will have the expected branch name 982 # For further information about this, see the commit message 983 command = ['branch', '-M', self.branch] 984 return self._dovccmd(command, self._initSubmodules)

985

986 - def _didFetch(self, res):

987 if self.revision: 988 head = self.revision 989 else: 990 head = 'FETCH_HEAD' 991 992 # That is not sufficient. git will leave unversioned files and empty 993 # directories. Clean them up manually in _didReset. 994 command = ['reset', '--hard', head] 995 return self._dovccmd(command, self._didHeadCheckout)

996 997 # Update first runs "git clean", removing local changes, 998 # if the branch to be checked out has changed. This, combined 999 # with the later "git reset" equates clobbering the repo, 1000 # but it's much more efficient.

1001 - def doVCUpdate(self):

1002 try: 1003 # Check to see if our branch has changed 1004 diffbranch = self.sourcedata != self.readSourcedata() 1005 except IOError: 1006 diffbranch = False 1007 if diffbranch: 1008 command = ['clean', '-f', '-d'] 1009 if self.ignore_ignores: 1010 command.append('-x') 1011 return self._dovccmd(command, self._didClean) 1012 return self._didClean(None)

1013

1014 - def _doFetch(self, dummy):

1015 # The plus will make sure the repo is moved to the branch's 1016 # head even if it is not a simple "fast-forward" 1017 command = ['fetch', '-t', self.repourl, '+%s' % self.branch] 1018 self.sendStatus({"header": "fetching branch %s from %s\n" 1019 % (self.branch, self.repourl)}) 1020 return self._dovccmd(command, self._didFetch)

1021

1022 - def _didClean(self, dummy):

1023 # After a clean, try to use the given revision if we have one. 1024 if self.revision: 1025 # We know what revision we want. See if we have it. 1026 d = self._dovccmd(['reset', '--hard', self.revision], 1027 self._initSubmodules) 1028 # If we are unable to reset to the specified version, we 1029 # must do a fetch first and retry. 1030 d.addErrback(self._doFetch) 1031 return d 1032 else: 1033 # No known revision, go grab the latest. 1034 return self._doFetch(None)

1035

1036 - def _didInit(self, res):

1037 return self.doVCUpdate()

1038

1039 - def doVCFull(self):

1040 # If they didn't ask for a specific revision, we can get away with a 1041 # shallow clone. 1042 if not self.args.get('revision') and self.args.get('shallow'): 1043 cmd = [self.vcexe, 'clone', '--depth', '1', self.repourl, 1044 self._fullSrcdir()] 1045 c = ShellCommand(self.builder, cmd, self.builder.basedir, 1046 sendRC=False, timeout=self.timeout, 1047 maxTime=self.maxTime, usePTY=False) 1048 self.command = c 1049 cmdexec = c.start() 1050 cmdexec.addCallback(self._didInit) 1051 return cmdexec 1052 else: 1053 os.makedirs(self._fullSrcdir()) 1054 return self._dovccmd(['init'], self._didInit)

1055

1056 - def parseGotRevision(self):

1057 command = ['rev-parse', 'HEAD'] 1058 def _parse(res): 1059 hash = self.command.stdout.strip() 1060 if len(hash) != 40: 1061 return None 1062 return hash

1063 return self._dovccmd(command, _parse, keepStdout=True)

1064 1065 registerSlaveCommand("git", Git, command_version) 1066

1067 -class Arch(SourceBase):

1068 """Arch-specific (tla-specific) VC operation. In addition to the 1069 arguments handled by SourceBase, this command reads the following keys: 1070 1071 ['url'] (required): the repository string 1072 ['version'] (required): which version (i.e. branch) to retrieve 1073 ['revision'] (optional): the 'patch-NN' argument to check out 1074 ['archive']: the archive name to use. If None, use the archive's default 1075 ['build-config']: if present, give to 'tla build-config' after checkout 1076 """ 1077 1078 header = "arch operation" 1079 buildconfig = None 1080

1081 - def setup(self, args):

1082 SourceBase.setup(self, args) 1083 self.vcexe = getCommand("tla") 1084 self.archive = args.get('archive') 1085 self.url = args['url'] 1086 self.version = args['version'] 1087 self.revision = args.get('revision') 1088 self.buildconfig = args.get('build-config') 1089 self.sourcedata = "%s\n%s\n%s\n" % (self.url, self.version, 1090 self.buildconfig)

1091

1092 - def sourcedirIsUpdateable(self):

1093 # Arch cannot roll a directory backwards, so if they ask for a 1094 # specific revision, clobber the directory. Technically this 1095 # could be limited to the cases where the requested revision is 1096 # later than our current one, but it's too hard to extract the 1097 # current revision from the tree. 1098 return (not self.revision and 1099 not self.sourcedirIsPatched() and 1100 os.path.isdir(os.path.join(self.builder.basedir, 1101 self.srcdir, "{arch}")))

1102

1103 - def doVCUpdate(self):

1104 # update: possible for mode in ('copy', 'update') 1105 d = os.path.join(self.builder.basedir, self.srcdir) 1106 command = [self.vcexe, 'replay'] 1107 if self.revision: 1108 command.append(self.revision) 1109 c = ShellCommand(self.builder, command, d, 1110 sendRC=False, timeout=self.timeout, 1111 maxTime=self.maxTime, usePTY=False) 1112 self.command = c 1113 return c.start()

1114

1115 - def doVCFull(self):

1116 # to do a checkout, we must first "register" the archive by giving 1117 # the URL to tla, which will go to the repository at that URL and 1118 # figure out the archive name. tla will tell you the archive name 1119 # when it is done, and all further actions must refer to this name. 1120 1121 command = [self.vcexe, 'register-archive', '--force', self.url] 1122 c = ShellCommand(self.builder, command, self.builder.basedir, 1123 sendRC=False, keepStdout=True, timeout=self.timeout, 1124 maxTime=self.maxTime, usePTY=False) 1125 self.command = c 1126 d = c.start() 1127 d.addCallback(self._abandonOnFailure) 1128 d.addCallback(self._didRegister, c) 1129 return d

1130

1131 - def _didRegister(self, res, c):

1132 # find out what tla thinks the archive name is. If the user told us 1133 # to use something specific, make sure it matches. 1134 r = re.search(r'Registering archive: (\S+)\s*$', c.stdout) 1135 if r: 1136 msg = "tla reports archive name is '%s'" % r.group(1) 1137 log.msg(msg) 1138 self.builder.sendUpdate({'header': msg+"\n"}) 1139 if self.archive and r.group(1) != self.archive: 1140 msg = (" mismatch, we wanted an archive named '%s'" 1141 % self.archive) 1142 log.msg(msg) 1143 self.builder.sendUpdate({'header': msg+"\n"}) 1144 raise AbandonChain(-1) 1145 self.archive = r.group(1) 1146 assert self.archive, "need archive name to continue" 1147 return self._doGet()

1148

1149 - def _doGet(self):

1150 ver = self.version 1151 if self.revision: 1152 ver += "--%s" % self.revision 1153 command = [self.vcexe, 'get', '--archive', self.archive, 1154 '--no-pristine', 1155 ver, self.srcdir] 1156 c = ShellCommand(self.builder, command, self.builder.basedir, 1157 sendRC=False, timeout=self.timeout, 1158 maxTime=self.maxTime, usePTY=False) 1159 self.command = c 1160 d = c.start() 1161 d.addCallback(self._abandonOnFailure) 1162 if self.buildconfig: 1163 d.addCallback(self._didGet) 1164 return d

1165

1166 - def _didGet(self, res):

1167 d = os.path.join(self.builder.basedir, self.srcdir) 1168 command = [self.vcexe, 'build-config', self.buildconfig] 1169 c = ShellCommand(self.builder, command, d, 1170 sendRC=False, timeout=self.timeout, 1171 maxTime=self.maxTime, usePTY=False) 1172 self.command = c 1173 d = c.start() 1174 d.addCallback(self._abandonOnFailure) 1175 return d

1176

1177 - def parseGotRevision(self):

1178 # using code from tryclient.TlaExtractor 1179 # 'tla logs --full' gives us ARCHIVE/BRANCH--REVISION 1180 # 'tla logs' gives us REVISION 1181 command = [self.vcexe, "logs", "--full", "--reverse"] 1182 c = ShellCommand(self.builder, command, 1183 os.path.join(self.builder.basedir, self.srcdir), 1184 environ=self.env, 1185 sendStdout=False, sendStderr=False, sendRC=False, 1186 keepStdout=True, usePTY=False) 1187 d = c.start() 1188 def _parse(res): 1189 tid = c.stdout.split("\n")[0].strip() 1190 slash = tid.index("/") 1191 dd = tid.rindex("--") 1192 #branch = tid[slash+1:dd] 1193 baserev = tid[dd+2:] 1194 return baserev

1195 d.addCallback(_parse) 1196 return d

1197 1198 registerSlaveCommand("arch", Arch, command_version) 1199

1200 -class Bazaar(Arch):

1201 """Bazaar (/usr/bin/baz) is an alternative client for Arch repositories. 1202 It is mostly option-compatible, but archive registration is different 1203 enough to warrant a separate Command. 1204 1205 ['archive'] (required): the name of the archive being used 1206 """ 1207

1208 - def setup(self, args):

1209 Arch.setup(self, args) 1210 self.vcexe = getCommand("baz") 1211 # baz doesn't emit the repository name after registration (and 1212 # grepping through the output of 'baz archives' is too hard), so we 1213 # require that the buildmaster configuration to provide both the 1214 # archive name and the URL. 1215 self.archive = args['archive'] # required for Baz 1216 self.sourcedata = "%s\n%s\n%s\n" % (self.url, self.version, 1217 self.buildconfig)

1218 1219 # in _didRegister, the regexp won't match, so we'll stick with the name 1220 # in self.archive 1221

1222 - def _doGet(self):

1223 # baz prefers ARCHIVE/VERSION. This will work even if 1224 # my-default-archive is not set. 1225 ver = self.archive + "/" + self.version 1226 if self.revision: 1227 ver += "--%s" % self.revision 1228 command = [self.vcexe, 'get', '--no-pristine', 1229 ver, self.srcdir] 1230 c = ShellCommand(self.builder, command, self.builder.basedir, 1231 sendRC=False, timeout=self.timeout, 1232 maxTime=self.maxTime, usePTY=False) 1233 self.command = c 1234 d = c.start() 1235 d.addCallback(self._abandonOnFailure) 1236 if self.buildconfig: 1237 d.addCallback(self._didGet) 1238 return d

1239

1240 - def parseGotRevision(self):

1241 # using code from tryclient.BazExtractor 1242 command = [self.vcexe, "tree-id"] 1243 c = ShellCommand(self.builder, command, 1244 os.path.join(self.builder.basedir, self.srcdir), 1245 environ=self.env, 1246 sendStdout=False, sendStderr=False, sendRC=False, 1247 keepStdout=True, usePTY=False) 1248 d = c.start() 1249 def _parse(res): 1250 tid = c.stdout.strip() 1251 slash = tid.index("/") 1252 dd = tid.rindex("--") 1253 #branch = tid[slash+1:dd] 1254 baserev = tid[dd+2:] 1255 return baserev

1256 d.addCallback(_parse) 1257 return d

1258 1259 registerSlaveCommand("bazaar", Bazaar, command_version) 1260 1261

1262 -class Bzr(SourceBase):

1263 """bzr-specific VC operation. In addition to the arguments 1264 handled by SourceBase, this command reads the following keys: 1265 1266 ['repourl'] (required): the Bzr repository string 1267 """ 1268 1269 header = "bzr operation" 1270

1271 - def setup(self, args):

1272 SourceBase.setup(self, args) 1273 self.vcexe = getCommand("bzr") 1274 self.repourl = args['repourl'] 1275 self.sourcedata = "%s\n" % self.repourl 1276 self.revision = self.args.get('revision') 1277 self.forceSharedRepo = args.get('forceSharedRepo')

1278

1279 - def sourcedirIsUpdateable(self):

1280 # checking out a specific revision requires a full 'bzr checkout' 1281 return (not self.revision and 1282 not self.sourcedirIsPatched() and 1283 os.path.isdir(os.path.join(self.builder.basedir, 1284 self.srcdir, ".bzr")))

1285

1286 - def start(self):

1287 def cont(res): 1288 # Continue with start() method in superclass. 1289 return SourceBase.start(self)

1290 1291 if self.forceSharedRepo: 1292 d = self.doForceSharedRepo(); 1293 d.addCallback(cont) 1294 return d 1295 else: 1296 return cont(None)

1297

1298 - def doVCUpdate(self):

1299 assert not self.revision 1300 # update: possible for mode in ('copy', 'update') 1301 srcdir = os.path.join(self.builder.basedir, self.srcdir) 1302 command = [self.vcexe, 'update'] 1303 c = ShellCommand(self.builder, command, srcdir, 1304 sendRC=False, timeout=self.timeout, 1305 maxTime=self.maxTime, usePTY=False) 1306 self.command = c 1307 return c.start()

1308

1309 - def doVCFull(self):

1310 # checkout or export 1311 d = self.builder.basedir 1312 if self.mode == "export": 1313 # exporting in bzr requires a separate directory 1314 return self.doVCExport() 1315 # originally I added --lightweight here, but then 'bzr revno' is 1316 # wrong. The revno reported in 'bzr version-info' is correct, 1317 # however. Maybe this is a bzr bug? 1318 # 1319 # In addition, you cannot perform a 'bzr update' on a repo pulled 1320 # from an HTTP repository that used 'bzr checkout --lightweight'. You 1321 # get a "ERROR: Cannot lock: transport is read only" when you try. 1322 # 1323 # So I won't bother using --lightweight for now. 1324 1325 command = [self.vcexe, 'checkout'] 1326 if self.revision: 1327 command.append('--revision') 1328 command.append(str(self.revision)) 1329 command.append(self.repourl) 1330 command.append(self.srcdir) 1331 1332 c = ShellCommand(self.builder, command, d, 1333 sendRC=False, timeout=self.timeout, 1334 maxTime=self.maxTime, usePTY=False) 1335 self.command = c 1336 d = c.start() 1337 return d

1338

1339 - def doVCExport(self):

1340 tmpdir = os.path.join(self.builder.basedir, "export-temp") 1341 srcdir = os.path.join(self.builder.basedir, self.srcdir) 1342 command = [self.vcexe, 'checkout', '--lightweight'] 1343 if self.revision: 1344 command.append('--revision') 1345 command.append(str(self.revision)) 1346 command.append(self.repourl) 1347 command.append(tmpdir) 1348 c = ShellCommand(self.builder, command, self.builder.basedir, 1349 sendRC=False, timeout=self.timeout, 1350 maxTime=self.maxTime, usePTY=False) 1351 self.command = c 1352 d = c.start() 1353 def _export(res): 1354 command = [self.vcexe, 'export', srcdir] 1355 c = ShellCommand(self.builder, command, tmpdir, 1356 sendRC=False, timeout=self.timeout, 1357 maxTime=self.maxTime, usePTY=False) 1358 self.command = c 1359 return c.start()

1360 d.addCallback(_export) 1361 return d 1362

1363 - def doForceSharedRepo(self):

1364 # Don't send stderr. When there is no shared repo, this might confuse 1365 # users, as they will see a bzr error message. But having no shared 1366 # repo is not an error, just an indication that we need to make one. 1367 c = ShellCommand(self.builder, [self.vcexe, 'info', '.'], 1368 self.builder.basedir, 1369 sendStderr=False, sendRC=False, usePTY=False) 1370 d = c.start() 1371 def afterCheckSharedRepo(res): 1372 if type(res) is int and res != 0: 1373 log.msg("No shared repo found, creating it") 1374 # bzr info fails, try to create shared repo. 1375 c = ShellCommand(self.builder, [self.vcexe, 'init-repo', '.'], 1376 self.builder.basedir, 1377 sendRC=False, usePTY=False) 1378 self.command = c 1379 return c.start() 1380 else: 1381 return defer.succeed(res)

1382 d.addCallback(afterCheckSharedRepo) 1383 return d 1384

1385 - def get_revision_number(self, out):

1386 # it feels like 'bzr revno' sometimes gives different results than 1387 # the 'revno:' line from 'bzr version-info', and the one from 1388 # version-info is more likely to be correct. 1389 for line in out.split("\n"): 1390 colon = line.find(":") 1391 if colon != -1: 1392 key, value = line[:colon], line[colon+2:] 1393 if key == "revno": 1394 return int(value) 1395 raise ValueError("unable to find revno: in bzr output: '%s'" % out)

1396

1397 - def parseGotRevision(self):

1398 command = [self.vcexe, "version-info"] 1399 c = ShellCommand(self.builder, command, 1400 os.path.join(self.builder.basedir, self.srcdir), 1401 environ=self.env, 1402 sendStdout=False, sendStderr=False, sendRC=False, 1403 keepStdout=True, usePTY=False) 1404 d = c.start() 1405 def _parse(res): 1406 try: 1407 return self.get_revision_number(c.stdout) 1408 except ValueError: 1409 msg =("Bzr.parseGotRevision unable to parse output " 1410 "of bzr version-info: '%s'" % c.stdout.strip()) 1411 log.msg(msg) 1412 self.sendStatus({'header': msg + "\n"}) 1413 return None

1414 d.addCallback(_parse) 1415 return d 1416 1417 registerSlaveCommand("bzr", Bzr, command_version) 1418

1419 -class Mercurial(SourceBase):

1420 """Mercurial specific VC operation. In addition to the arguments 1421 handled by SourceBase, this command reads the following keys: 1422 1423 ['repourl'] (required): the Mercurial repository string 1424 ['clobberOnBranchChange']: Document me. See ticket #462. 1425 """ 1426 1427 header = "mercurial operation" 1428

1429 - def setup(self, args):

1430 SourceBase.setup(self, args) 1431 self.vcexe = getCommand("hg") 1432 self.repourl = args['repourl'] 1433 self.clobberOnBranchChange = args.get('clobberOnBranchChange', True) 1434 self.sourcedata = "%s\n" % self.repourl 1435 self.branchType = args.get('branchType', 'dirname') 1436 self.stdout = "" 1437 self.stderr = "" 1438 self.clobbercount = 0 # n times we've clobbered

1439

1440 - def sourcedirIsUpdateable(self):

1441 return os.path.isdir(os.path.join(self.builder.basedir, 1442 self.srcdir, ".hg"))

1443

1444 - def doVCUpdate(self):

1445 d = os.path.join(self.builder.basedir, self.srcdir) 1446 command = [self.vcexe, 'pull', '--verbose', self.repourl] 1447 c = ShellCommand(self.builder, command, d, 1448 sendRC=False, timeout=self.timeout, 1449 maxTime=self.maxTime, keepStdout=True, usePTY=False) 1450 self.command = c 1451 d = c.start() 1452 d.addCallback(self._handleEmptyUpdate) 1453 d.addCallback(self._update) 1454 return d

1455

1456 - def _handleEmptyUpdate(self, res):

1457 if type(res) is int and res == 1: 1458 if self.command.stdout.find("no changes found") != -1: 1459 # 'hg pull', when it doesn't have anything to do, exits with 1460 # rc=1, and there appears to be no way to shut this off. It 1461 # emits a distinctive message to stdout, though. So catch 1462 # this and pretend that it completed successfully. 1463 return 0 1464 return res

1465

1466 - def doVCFull(self):

1467 d = os.path.join(self.builder.basedir, self.srcdir) 1468 command = [self.vcexe, 'clone', '--verbose', '--noupdate'] 1469 1470 # if got revision, clobbering and in dirname, only clone to specific revision 1471 # (otherwise, do full clone to re-use .hg dir for subsequent builds) 1472 if self.args.get('revision') and self.mode == 'clobber' and self.branchType == 'dirname': 1473 command.extend(['--rev', self.args.get('revision')]) 1474 command.extend([self.repourl, d]) 1475 1476 c = ShellCommand(self.builder, command, self.builder.basedir, 1477 sendRC=False, timeout=self.timeout, 1478 maxTime=self.maxTime, usePTY=False) 1479 self.command = c 1480 cmd1 = c.start() 1481 cmd1.addCallback(self._update) 1482 return cmd1

1483

1484 - def _clobber(self, dummy, dirname):

1485 self.clobbercount += 1 1486 1487 if self.clobbercount > 3: 1488 raise Exception, "Too many clobber attempts. Aborting step" 1489 1490 def _vcfull(res): 1491 return self.doVCFull()

1492 1493 c = self.doClobber(dummy, dirname) 1494 c.addCallback(_vcfull) 1495 1496 return c

1497

1498 - def _purge(self, dummy, dirname):

1499 d = os.path.join(self.builder.basedir, self.srcdir) 1500 purge = [self.vcexe, 'purge', '--all'] 1501 purgeCmd = ShellCommand(self.builder, purge, d, 1502 sendStdout=False, sendStderr=False, 1503 keepStdout=True, keepStderr=True, usePTY=False) 1504 1505 def _clobber(res): 1506 if res != 0: 1507 # purge failed, we need to switch to a classic clobber 1508 msg = "'hg purge' failed: %s\n%s. Clobbering." % (purgeCmd.stdout, purgeCmd.stderr) 1509 self.sendStatus({'header': msg + "\n"}) 1510 log.msg(msg) 1511 1512 return self._clobber(dummy, dirname) 1513 1514 # Purge was a success, then we need to update 1515 return self._update2(res)

1516 1517 p = purgeCmd.start() 1518 p.addCallback(_clobber) 1519 return p 1520

1521 - def _update(self, res):

1522 if res != 0: 1523 return res 1524 1525 # compare current branch to update 1526 self.update_branch = self.args.get('branch', 'default') 1527 1528 d = os.path.join(self.builder.basedir, self.srcdir) 1529 parentscmd = [self.vcexe, 'identify', '--num', '--branch'] 1530 cmd = ShellCommand(self.builder, parentscmd, d, 1531 sendStdout=False, sendStderr=False, 1532 keepStdout=True, keepStderr=True, usePTY=False) 1533 1534 self.clobber = None 1535 1536 def _parseIdentify(res): 1537 if res != 0: 1538 msg = "'hg identify' failed: %s\n%s" % (cmd.stdout, cmd.stderr) 1539 self.sendStatus({'header': msg + "\n"}) 1540 log.msg(msg) 1541 return res 1542 1543 log.msg('Output: %s' % cmd.stdout) 1544 1545 match = re.search(r'^(.+) (.+)$', cmd.stdout) 1546 assert match 1547 1548 rev = match.group(1) 1549 current_branch = match.group(2) 1550 1551 if rev == '-1': 1552 msg = "Fresh hg repo, don't worry about in-repo branch name" 1553 log.msg(msg) 1554 1555 elif self.sourcedirIsPatched(): 1556 self.clobber = self._purge 1557 1558 elif self.update_branch != current_branch: 1559 msg = "Working dir is on in-repo branch '%s' and build needs '%s'." % (current_branch, self.update_branch) 1560 if self.clobberOnBranchChange: 1561 msg += ' Cloberring.' 1562 else: 1563 msg += ' Updating.' 1564 1565 self.sendStatus({'header': msg + "\n"}) 1566 log.msg(msg) 1567 1568 # Clobbers only if clobberOnBranchChange is set 1569 if self.clobberOnBranchChange: 1570 self.clobber = self._purge 1571 1572 else: 1573 msg = "Working dir on same in-repo branch as build (%s)." % (current_branch) 1574 log.msg(msg) 1575 1576 return 0

1577 1578 def _checkRepoURL(res): 1579 parentscmd = [self.vcexe, 'paths', 'default'] 1580 cmd2 = ShellCommand(self.builder, parentscmd, d, 1581 sendStdout=False, sendStderr=False, 1582 keepStdout=True, keepStderr=True, usePTY=False) 1583 1584 def _parseRepoURL(res): 1585 if res == 1: 1586 if "not found!" == cmd2.stderr.strip(): 1587 msg = "hg default path not set. Not checking repo url for clobber test" 1588 log.msg(msg) 1589 return 0 1590 else: 1591 msg = "'hg paths default' failed: %s\n%s" % (cmd2.stdout, cmd2.stderr) 1592 log.msg(msg) 1593 return 1 1594 1595 oldurl = cmd2.stdout.strip() 1596 1597 log.msg("Repo cloned from: '%s'" % oldurl) 1598 1599 if sys.platform == "win32": 1600 oldurl = oldurl.lower().replace('\\', '/') 1601 repourl = self.repourl.lower().replace('\\', '/') 1602 else: 1603 repourl = self.repourl 1604 1605 if repourl.startswith('file://'): 1606 repourl = repourl.split('file://')[1] 1607 if oldurl.startswith('file://'): 1608 oldurl = oldurl.split('file://')[1] 1609 1610 oldurl = remove_userpassword(oldurl) 1611 repourl = remove_userpassword(repourl) 1612 1613 if oldurl.rstrip('/') != repourl.rstrip('/'): 1614 self.clobber = self._clobber 1615 msg = "RepoURL changed from '%s' in wc to '%s' in update. Clobbering" % (oldurl, repourl) 1616 log.msg(msg) 1617 1618 return 0 1619 1620 c = cmd2.start() 1621 c.addCallback(_parseRepoURL) 1622 return c 1623 1624 def _maybeClobber(res): 1625 if self.clobber: 1626 msg = "Clobber flag set. Doing clobbering" 1627 log.msg(msg) 1628 1629 def _vcfull(res): 1630 return self.doVCFull() 1631 1632 return self.clobber(None, self.srcdir) 1633 1634 return 0 1635 1636 c = cmd.start() 1637 c.addCallback(_parseIdentify) 1638 c.addCallback(_checkRepoURL) 1639 c.addCallback(_maybeClobber) 1640 c.addCallback(self._update2) 1641 return c 1642

1643 - def _update2(self, res):

1644 d = os.path.join(self.builder.basedir, self.srcdir) 1645 1646 updatecmd=[self.vcexe, 'update', '--clean', '--repository', d] 1647 if self.args.get('revision'): 1648 updatecmd.extend(['--rev', self.args['revision']]) 1649 else: 1650 updatecmd.extend(['--rev', self.args.get('branch', 'default')]) 1651 self.command = ShellCommand(self.builder, updatecmd, 1652 self.builder.basedir, sendRC=False, 1653 timeout=self.timeout, maxTime=self.maxTime, usePTY=False) 1654 return self.command.start()

1655

1656 - def parseGotRevision(self):

1657 # we use 'hg identify' to find out what we wound up with 1658 command = [self.vcexe, "identify", "--id", "--debug"] # get full rev id 1659 c = ShellCommand(self.builder, command, 1660 os.path.join(self.builder.basedir, self.srcdir), 1661 environ=self.env, 1662 sendStdout=False, sendStderr=False, sendRC=False, 1663 keepStdout=True, usePTY=False) 1664 d = c.start() 1665 def _parse(res): 1666 m = re.search(r'^(\w+)', c.stdout) 1667 return m.group(1)

1668 d.addCallback(_parse) 1669 return d 1670 1671 registerSlaveCommand("hg", Mercurial, command_version) 1672 1673

1674 -class P4Base(SourceBase):

1675 """Base class for P4 source-updaters 1676 1677 ['p4port'] (required): host:port for server to access 1678 ['p4user'] (optional): user to use for access 1679 ['p4passwd'] (optional): passwd to try for the user 1680 ['p4client'] (optional): client spec to use 1681 """

1682 - def setup(self, args):

1683 SourceBase.setup(self, args) 1684 self.p4port = args['p4port'] 1685 self.p4client = args['p4client'] 1686 self.p4user = args['p4user'] 1687 self.p4passwd = args['p4passwd']

1688

1689 - def parseGotRevision(self):

1690 # Executes a p4 command that will give us the latest changelist number 1691 # of any file under the current (or default) client: 1692 command = ['p4'] 1693 if self.p4port: 1694 command.extend(['-p', self.p4port]) 1695 if self.p4user: 1696 command.extend(['-u', self.p4user]) 1697 if self.p4passwd: 1698 command.extend(['-P', Obfuscated(self.p4passwd, "XXXXXXXX")]) 1699 if self.p4client: 1700 command.extend(['-c', self.p4client]) 1701 # add '-s submitted' for bug #626 1702 command.extend(['changes', '-s', 'submitted', '-m', '1', '#have']) 1703 c = ShellCommand(self.builder, command, self.builder.basedir, 1704 environ=self.env, timeout=self.timeout, 1705 maxTime=self.maxTime, sendStdout=True, 1706 sendStderr=False, sendRC=False, keepStdout=True, 1707 usePTY=False) 1708 self.command = c 1709 d = c.start() 1710 1711 def _parse(res): 1712 # 'p4 -c clien-name change -m 1 "#have"' will produce an output like: 1713 # "Change 28147 on 2008/04/07 by p4user@hostname..." 1714 # The number after "Change" is the one we want. 1715 m = re.match('Change\s+(\d+)\s+', c.stdout) 1716 if m: 1717 return m.group(1) 1718 return None

1719 d.addCallback(_parse) 1720 return d

1721 1722

1723 -class P4(P4Base):

1724 """A P4 source-updater. 1725 1726 ['p4port'] (required): host:port for server to access 1727 ['p4user'] (optional): user to use for access 1728 ['p4passwd'] (optional): passwd to try for the user 1729 ['p4client'] (optional): client spec to use 1730 ['p4extra_views'] (optional): additional client views to use 1731 """ 1732 1733 header = "p4" 1734

1735 - def setup(self, args):

1736 P4Base.setup(self, args) 1737 self.p4base = args['p4base'] 1738 self.p4extra_views = args['p4extra_views'] 1739 self.p4mode = args['mode'] 1740 self.p4branch = args['branch'] 1741 1742 self.sourcedata = str([ 1743 # Perforce server. 1744 self.p4port, 1745 1746 # Client spec. 1747 self.p4client, 1748 1749 # Depot side of view spec. 1750 self.p4base, 1751 self.p4branch, 1752 self.p4extra_views, 1753 1754 # Local side of view spec (srcdir is made from these). 1755 self.builder.basedir, 1756 self.mode, 1757 self.workdir 1758 ])

1759 1760

1761 - def sourcedirIsUpdateable(self):

1762 # We assume our client spec is still around. 1763 # We just say we aren't updateable if the dir doesn't exist so we 1764 # don't get ENOENT checking the sourcedata. 1765 return (not self.sourcedirIsPatched() and 1766 os.path.isdir(os.path.join(self.builder.basedir, 1767 self.srcdir)))

1768

1769 - def doVCUpdate(self):

1770 return self._doP4Sync(force=False)

1771

1772 - def _doP4Sync(self, force):

1773 command = ['p4'] 1774 1775 if self.p4port: 1776 command.extend(['-p', self.p4port]) 1777 if self.p4user: 1778 command.extend(['-u', self.p4user]) 1779 if self.p4passwd: 1780 command.extend(['-P', Obfuscated(self.p4passwd, "XXXXXXXX")]) 1781 if self.p4client: 1782 command.extend(['-c', self.p4client]) 1783 command.extend(['sync']) 1784 if force: 1785 command.extend(['-f']) 1786 if self.revision: 1787 command.extend(['@' + str(self.revision)]) 1788 env = {} 1789 c = ShellCommand(self.builder, command, self.builder.basedir, 1790 environ=env, sendRC=False, timeout=self.timeout, 1791 maxTime=self.maxTime, keepStdout=True, usePTY=False) 1792 self.command = c 1793 d = c.start() 1794 d.addCallback(self._abandonOnFailure) 1795 return d

1796 1797

1798 - def doVCFull(self):

1799 env = {} 1800 command = ['p4'] 1801 client_spec = '' 1802 client_spec += "Client: %s\n\n" % self.p4client 1803 client_spec += "Owner: %s\n\n" % self.p4user 1804 client_spec += "Description:\n\tCreated by %s\n\n" % self.p4user 1805 client_spec += "Root:\t%s\n\n" % self.builder.basedir 1806 client_spec += "Options:\tallwrite rmdir\n\n" 1807 client_spec += "LineEnd:\tlocal\n\n" 1808 1809 # Setup a view 1810 client_spec += "View:\n\t%s" % (self.p4base) 1811 if self.p4branch: 1812 client_spec += "%s/" % (self.p4branch) 1813 client_spec += "... //%s/%s/...\n" % (self.p4client, self.srcdir) 1814 if self.p4extra_views: 1815 for k, v in self.p4extra_views: 1816 client_spec += "\t%s/... //%s/%s%s/...\n" % (k, self.p4client, 1817 self.srcdir, v) 1818 if self.p4port: 1819 command.extend(['-p', self.p4port]) 1820 if self.p4user: 1821 command.extend(['-u', self.p4user]) 1822 if self.p4passwd: 1823 command.extend(['-P', Obfuscated(self.p4passwd, "XXXXXXXX")]) 1824 command.extend(['client', '-i']) 1825 log.msg(client_spec) 1826 1827 # from bdbaddog in github comments: 1828 # I'm pretty sure the issue is that perforce client specs can't be 1829 # non-ascii (unless you configure at initial config to be unicode). I 1830 # floated a question to perforce mailing list. From reading the 1831 # internationalization notes.. 1832 # http://www.perforce.com/perforce/doc.092/user/i18nnotes.txt 1833 # I'm 90% sure that's the case. 1834 # (http://github.com/bdbaddog/buildbot/commit/8420149b2b804efcf5f81a13e18aa62da0424d21) 1835 1836 # Clean client spec to plain ascii 1837 client_spec=client_spec.encode('ascii','ignore') 1838 1839 c = ShellCommand(self.builder, command, self.builder.basedir, 1840 environ=env, sendRC=False, timeout=self.timeout, 1841 maxTime=self.maxTime, initialStdin=client_spec, 1842 usePTY=False) 1843 self.command = c 1844 d = c.start() 1845 d.addCallback(self._abandonOnFailure) 1846 d.addCallback(lambda _: self._doP4Sync(force=True)) 1847 return d

1848

1849 - def parseGotRevision(self):

1850 if self.revision: 1851 return str(self.revision) 1852 else: 1853 return P4Base.parseGotRevision(self)

1854 1855 registerSlaveCommand("p4", P4, command_version) 1856 1857

1858 -class P4Sync(P4Base):

1859 """A partial P4 source-updater. Requires manual setup of a per-slave P4 1860 environment. The only thing which comes from the master is P4PORT. 1861 'mode' is required to be 'copy'. 1862 1863 ['p4port'] (required): host:port for server to access 1864 ['p4user'] (optional): user to use for access 1865 ['p4passwd'] (optional): passwd to try for the user 1866 ['p4client'] (optional): client spec to use 1867 """ 1868 1869 header = "p4 sync" 1870

1871 - def setup(self, args):

1872 P4Base.setup(self, args) 1873 self.vcexe = getCommand("p4")

1874

1875 - def sourcedirIsUpdateable(self):

1876 return True

1877

1878 - def _doVC(self, force):

1879 d = os.path.join(self.builder.basedir, self.srcdir) 1880 command = [self.vcexe] 1881 if self.p4port: 1882 command.extend(['-p', self.p4port]) 1883 if self.p4user: 1884 command.extend(['-u', self.p4user]) 1885 if self.p4passwd: 1886 command.extend(['-P', Obfuscated(self.p4passwd, "XXXXXXXX")]) 1887 if self.p4client: 1888 command.extend(['-c', self.p4client]) 1889 command.extend(['sync']) 1890 if force: 1891 command.extend(['-f']) 1892 if self.revision: 1893 command.extend(['@' + self.revision]) 1894 env = {} 1895 c = ShellCommand(self.builder, command, d, environ=env, 1896 sendRC=False, timeout=self.timeout, 1897 maxTime=self.maxTime, usePTY=False) 1898 self.command = c 1899 return c.start()

1900

1901 - def doVCUpdate(self):

1902 return self._doVC(force=False)

1903

1904 - def doVCFull(self):

1905 return self._doVC(force=True)

1906

1907 - def parseGotRevision(self):

1908 if self.revision: 1909 return str(self.revision) 1910 else: 1911 return P4Base.parseGotRevision(self)

1912 1913 registerSlaveCommand("p4sync", P4Sync, command_version) 1914

Source Code for Module buildbot.slave.commands.vcs