Package buildslave :: Package commands :: Module vcs
[frames] | no frames]

Source Code for Module buildslave.commands.vcs

   1  import os, re, sys, shutil, time 
   2   
   3  from xml.dom.minidom import parseString 
   4   
   5  from twisted.python import log, failure, runtime 
   6  from twisted.internet import defer 
   7   
   8  from buildslave.commands.base import Command, ShellCommand, AbandonChain, command_version, Obfuscated 
   9  from buildslave.commands.registry import registerSlaveCommand 
  10  from buildslave.commands.utils import getCommand, rmdirRecursive 
  11  from buildslave.util import remove_userpassword 
  12   
13 -class SourceBase(Command):
14 """Abstract base class for Version Control System operations (checkout 15 and update). This class extracts the following arguments from the 16 dictionary received from the master: 17 18 - ['workdir']: (required) the subdirectory where the buildable sources 19 should be placed 20 21 - ['mode']: one of update/copy/clobber/export, defaults to 'update' 22 23 - ['revision']: If not None, this is an int or string which indicates 24 which sources (along a time-like axis) should be used. 25 It is the thing you provide as the CVS -r or -D 26 argument. 27 28 - ['patch']: If not None, this is a tuple of (striplevel, patch) 29 which contains a patch that should be applied after the 30 checkout has occurred. Once applied, the tree is no 31 longer eligible for use with mode='update', and it only 32 makes sense to use this in conjunction with a 33 ['revision'] argument. striplevel is an int, and patch 34 is a string in standard unified diff format. The patch 35 will be applied with 'patch -p%d <PATCH', with 36 STRIPLEVEL substituted as %d. The command will fail if 37 the patch process fails (rejected hunks). 38 39 - ['timeout']: seconds of silence tolerated before we kill off the 40 command 41 42 - ['maxTime']: seconds before we kill off the command 43 44 - ['retry']: If not None, this is a tuple of (delay, repeats) 45 which means that any failed VC updates should be 46 reattempted, up to REPEATS times, after a delay of 47 DELAY seconds. This is intended to deal with slaves 48 that experience transient network failures. 49 """ 50 51 sourcedata = "" 52
53 - def setup(self, args):
54 # if we need to parse the output, use this environment. Otherwise 55 # command output will be in whatever the buildslave's native language 56 # has been set to. 57 self.env = os.environ.copy() 58 self.env['LC_MESSAGES'] = "C" 59 60 self.workdir = args['workdir'] 61 self.mode = args.get('mode', "update") 62 self.revision = args.get('revision') 63 self.patch = args.get('patch') 64 self.timeout = args.get('timeout', 120) 65 self.maxTime = args.get('maxTime', None) 66 self.retry = args.get('retry')
67 # VC-specific subclasses should override this to extract more args. 68 # Make sure to upcall! 69
70 - def start(self):
71 self.sendStatus({'header': "starting " + self.header + "\n"}) 72 self.command = None 73 74 # self.srcdir is where the VC system should put the sources 75 if self.mode == "copy": 76 self.srcdir = "source" # hardwired directory name, sorry 77 else: 78 self.srcdir = self.workdir 79 self.sourcedatafile = os.path.join(self.builder.basedir, 80 self.srcdir, 81 ".buildbot-sourcedata") 82 83 d = defer.succeed(None) 84 self.maybeClobber(d) 85 if not (self.sourcedirIsUpdateable() and self.sourcedataMatches()): 86 # the directory cannot be updated, so we have to clobber it. 87 # Perhaps the master just changed modes from 'export' to 88 # 'update'. 89 d.addCallback(self.doClobber, self.srcdir) 90 91 d.addCallback(self.doVC) 92 93 if self.mode == "copy": 94 d.addCallback(self.doCopy) 95 if self.patch: 96 d.addCallback(self.doPatch) 97 d.addCallbacks(self._sendRC, self._checkAbandoned) 98 return d
99
100 - def maybeClobber(self, d):
101 # do we need to clobber anything? 102 if self.mode in ("copy", "clobber", "export"): 103 d.addCallback(self.doClobber, self.workdir)
104
105 - def interrupt(self):
106 self.interrupted = True 107 if self.command: 108 self.command.kill("command interrupted")
109
110 - def doVC(self, res):
111 if self.interrupted: 112 raise AbandonChain(1) 113 if self.sourcedirIsUpdateable() and self.sourcedataMatches(): 114 d = self.doVCUpdate() 115 d.addCallback(self.maybeDoVCFallback) 116 else: 117 d = self.doVCFull() 118 d.addBoth(self.maybeDoVCRetry) 119 d.addCallback(self._abandonOnFailure) 120 d.addCallback(self._handleGotRevision) 121 d.addCallback(self.writeSourcedata) 122 return d
123
124 - def sourcedataMatches(self):
125 try: 126 olddata = self.readSourcedata() 127 if olddata != self.sourcedata: 128 return False 129 except IOError: 130 return False 131 return True
132
133 - def sourcedirIsPatched(self):
134 return os.path.exists(os.path.join(self.builder.basedir, 135 self.workdir, 136 ".buildbot-patched"))
137
138 - def _handleGotRevision(self, res):
139 d = defer.maybeDeferred(self.parseGotRevision) 140 d.addCallback(lambda got_revision: 141 self.sendStatus({'got_revision': got_revision})) 142 return d
143
144 - def parseGotRevision(self):
145 """Override this in a subclass. It should return a string that 146 represents which revision was actually checked out, or a Deferred 147 that will fire with such a string. If, in a future build, you were to 148 pass this 'got_revision' string in as the 'revision' component of a 149 SourceStamp, you should wind up with the same source code as this 150 checkout just obtained. 151 152 It is probably most useful to scan self.command.stdout for a string 153 of some sort. Be sure to set keepStdout=True on the VC command that 154 you run, so that you'll have something available to look at. 155 156 If this information is unavailable, just return None.""" 157 158 return None
159
160 - def readSourcedata(self):
161 return open(self.sourcedatafile, "r").read()
162
163 - def writeSourcedata(self, res):
164 open(self.sourcedatafile, "w").write(self.sourcedata) 165 return res
166
167 - def sourcedirIsUpdateable(self):
168 """Returns True if the tree can be updated.""" 169 raise NotImplementedError("this must be implemented in a subclass")
170
171 - def doVCUpdate(self):
172 """Returns a deferred with the steps to update a checkout.""" 173 raise NotImplementedError("this must be implemented in a subclass")
174
175 - def doVCFull(self):
176 """Returns a deferred with the steps to do a fresh checkout.""" 177 raise NotImplementedError("this must be implemented in a subclass")
178
179 - def maybeDoVCFallback(self, rc):
180 if type(rc) is int and rc == 0: 181 return rc 182 if self.interrupted: 183 raise AbandonChain(1) 184 msg = "update failed, clobbering and trying again" 185 self.sendStatus({'header': msg + "\n"}) 186 log.msg(msg) 187 d = self.doClobber(None, self.srcdir) 188 d.addCallback(self.doVCFallback2) 189 return d
190
191 - def doVCFallback2(self, res):
192 msg = "now retrying VC operation" 193 self.sendStatus({'header': msg + "\n"}) 194 log.msg(msg) 195 d = self.doVCFull() 196 d.addBoth(self.maybeDoVCRetry) 197 d.addCallback(self._abandonOnFailure) 198 return d
199
200 - def maybeDoVCRetry(self, res):
201 """We get here somewhere after a VC chain has finished. res could 202 be:: 203 204 - 0: the operation was successful 205 - nonzero: the operation failed. retry if possible 206 - AbandonChain: the operation failed, someone else noticed. retry. 207 - Failure: some other exception, re-raise 208 """ 209 210 if isinstance(res, failure.Failure): 211 if self.interrupted: 212 return res # don't re-try interrupted builds 213 res.trap(AbandonChain) 214 else: 215 if type(res) is int and res == 0: 216 return res 217 if self.interrupted: 218 raise AbandonChain(1) 219 # if we get here, we should retry, if possible 220 if self.retry: 221 delay, repeats = self.retry 222 if repeats >= 0: 223 self.retry = (delay, repeats-1) 224 msg = ("update failed, trying %d more times after %d seconds" 225 % (repeats, delay)) 226 self.sendStatus({'header': msg + "\n"}) 227 log.msg(msg) 228 d = defer.Deferred() 229 # we are going to do a full checkout, so a clobber is 230 # required first 231 self.doClobber(d, self.workdir) 232 if self.srcdir: 233 self.doClobber(d, self.srcdir) 234 d.addCallback(lambda res: self.doVCFull()) 235 d.addBoth(self.maybeDoVCRetry) 236 self._reactor.callLater(delay, d.callback, None) 237 return d 238 return res
239
240 - def doClobber(self, dummy, dirname, chmodDone=False):
241 # TODO: remove the old tree in the background 242 ## workdir = os.path.join(self.builder.basedir, self.workdir) 243 ## deaddir = self.workdir + ".deleting" 244 ## if os.path.isdir(workdir): 245 ## try: 246 ## os.rename(workdir, deaddir) 247 ## # might fail if deaddir already exists: previous deletion 248 ## # hasn't finished yet 249 ## # start the deletion in the background 250 ## # TODO: there was a solaris/NetApp/NFS problem where a 251 ## # process that was still running out of the directory we're 252 ## # trying to delete could prevent the rm-rf from working. I 253 ## # think it stalled the rm, but maybe it just died with 254 ## # permission issues. Try to detect this. 255 ## os.commands("rm -rf %s &" % deaddir) 256 ## except: 257 ## # fall back to sequential delete-then-checkout 258 ## pass 259 d = os.path.join(self.builder.basedir, dirname) 260 if runtime.platformType != "posix": 261 # if we're running on w32, use rmtree instead. It will block, 262 # but hopefully it won't take too long. 263 rmdirRecursive(d) 264 return defer.succeed(0) 265 command = ["rm", "-rf", d] 266 c = ShellCommand(self.builder, command, self.builder.basedir, 267 sendRC=0, timeout=self.timeout, maxTime=self.maxTime, 268 usePTY=False) 269 270 self.command = c 271 # sendRC=0 means the rm command will send stdout/stderr to the 272 # master, but not the rc=0 when it finishes. That job is left to 273 # _sendRC 274 d = c.start() 275 # The rm -rf may fail if there is a left-over subdir with chmod 000 276 # permissions. So if we get a failure, we attempt to chmod suitable 277 # permissions and re-try the rm -rf. 278 if chmodDone: 279 d.addCallback(self._abandonOnFailure) 280 else: 281 d.addCallback(lambda rc: self.doClobberTryChmodIfFail(rc, dirname)) 282 return d
283
284 - def doClobberTryChmodIfFail(self, rc, dirname):
285 assert isinstance(rc, int) 286 if rc == 0: 287 return defer.succeed(0) 288 # Attempt a recursive chmod and re-try the rm -rf after. 289 290 command = ["chmod", "-Rf", "u+rwx", os.path.join(self.builder.basedir, dirname)] 291 if sys.platform.startswith('freebsd'): 292 # Work around a broken 'chmod -R' on FreeBSD (it tries to recurse into a 293 # directory for which it doesn't have permission, before changing that 294 # permission) by running 'find' instead 295 command = ["find", os.path.join(self.builder.basedir, dirname), 296 '-exec', 'chmod', 'u+rwx', '{}', ';' ] 297 c = ShellCommand(self.builder, command, self.builder.basedir, 298 sendRC=0, timeout=self.timeout, maxTime=self.maxTime, 299 usePTY=False) 300 301 self.command = c 302 d = c.start() 303 d.addCallback(self._abandonOnFailure) 304 d.addCallback(lambda dummy: self.doClobber(dummy, dirname, True)) 305 return d
306
307 - def doCopy(self, res):
308 # now copy tree to workdir 309 fromdir = os.path.join(self.builder.basedir, self.srcdir) 310 todir = os.path.join(self.builder.basedir, self.workdir) 311 if runtime.platformType != "posix": 312 self.sendStatus({'header': "Since we're on a non-POSIX platform, " 313 "we're not going to try to execute cp in a subprocess, but instead " 314 "use shutil.copytree(), which will block until it is complete. " 315 "fromdir: %s, todir: %s\n" % (fromdir, todir)}) 316 shutil.copytree(fromdir, todir) 317 return defer.succeed(0) 318 319 if not os.path.exists(os.path.dirname(todir)): 320 os.makedirs(os.path.dirname(todir)) 321 if os.path.exists(todir): 322 # I don't think this happens, but just in case.. 323 log.msg("cp target '%s' already exists -- cp will not do what you think!" % todir) 324 325 command = ['cp', '-R', '-P', '-p', fromdir, todir] 326 c = ShellCommand(self.builder, command, self.builder.basedir, 327 sendRC=False, timeout=self.timeout, maxTime=self.maxTime, 328 usePTY=False) 329 self.command = c 330 d = c.start() 331 d.addCallback(self._abandonOnFailure) 332 return d
333
334 - def doPatch(self, res):
335 patchlevel = self.patch[0] 336 diff = self.patch[1] 337 root = None 338 if len(self.patch) >= 3: 339 root = self.patch[2] 340 command = [ 341 getCommand("patch"), 342 '-p%d' % patchlevel, 343 '--remove-empty-files', 344 '--force', 345 '--forward', 346 ] 347 dir = os.path.join(self.builder.basedir, self.workdir) 348 # Mark the directory so we don't try to update it later, or at least try 349 # to revert first. 350 marker = open(os.path.join(dir, ".buildbot-patched"), "w") 351 marker.write("patched\n") 352 marker.close() 353 354 # Update 'dir' with the 'root' option. Make sure it is a subdirectory 355 # of dir. 356 if (root and 357 os.path.abspath(os.path.join(dir, root) 358 ).startswith(os.path.abspath(dir))): 359 dir = os.path.join(dir, root) 360 361 # now apply the patch 362 c = ShellCommand(self.builder, command, dir, 363 sendRC=False, timeout=self.timeout, 364 maxTime=self.maxTime, initialStdin=diff, usePTY=False) 365 self.command = c 366 d = c.start() 367 d.addCallback(self._abandonOnFailure) 368 return d
369 370 371
372 -class BK(SourceBase):
373 """BitKeeper-specific VC operation. In addition to the arguments 374 handled by SourceBase, this command reads the following keys: 375 376 ['bkurl'] (required): the BK repository string 377 """ 378 379 header = "bk operation" 380
381 - def setup(self, args):
382 SourceBase.setup(self, args) 383 self.vcexe = getCommand("bk") 384 self.bkurl = args['bkurl'] 385 self.sourcedata = '"%s\n"' % self.bkurl 386 387 self.bk_args = [] 388 if args.get('extra_args', None) is not None: 389 self.bk_args.extend(args['extra_args'])
390
391 - def sourcedirIsUpdateable(self):
392 if os.path.exists(os.path.join(self.builder.basedir, 393 self.srcdir, ".buildbot-patched")): 394 return False 395 return os.path.isfile(os.path.join(self.builder.basedir, 396 self.srcdir, "BK/parent"))
397
398 - def doVCUpdate(self):
399 revision = self.args['revision'] or 'HEAD' 400 # update: possible for mode in ('copy', 'update') 401 d = os.path.join(self.builder.basedir, self.srcdir) 402 403 # Revision is ignored since the BK free client doesn't support it. 404 command = [self.vcexe, 'pull'] 405 c = ShellCommand(self.builder, command, d, 406 sendRC=False, timeout=self.timeout, 407 keepStdout=True, usePTY=False) 408 self.command = c 409 return c.start()
410
411 - def doVCFull(self):
412 413 revision_arg = '' 414 if self.args['revision']: 415 revision_arg = "-r%s" % self.args['revision'] 416 417 d = self.builder.basedir 418 419 command = [self.vcexe, 'clone', revision_arg] + self.bk_args + \ 420 [self.bkurl, self.srcdir] 421 c = ShellCommand(self.builder, command, d, 422 sendRC=False, timeout=self.timeout, 423 keepStdout=True, usePTY=False) 424 self.command = c 425 return c.start()
426
427 - def getBKVersionCommand(self):
428 """ 429 Get the (shell) command used to determine BK revision number 430 of checked-out code 431 432 return: list of strings, passable as the command argument to ShellCommand 433 """ 434 return [self.vcexe, "changes", "-r+", "-d:REV:"]
435
436 - def parseGotRevision(self):
437 c = ShellCommand(self.builder, 438 self.getBKVersionCommand(), 439 os.path.join(self.builder.basedir, self.srcdir), 440 environ=self.env, 441 sendStdout=False, sendStderr=False, sendRC=False, 442 keepStdout=True, usePTY=False) 443 d = c.start() 444 def _parse(res): 445 r_raw = c.stdout.strip() 446 got_version = None 447 try: 448 r = r_raw 449 except: 450 msg = ("BK.parseGotRevision unable to parse output: (%s)" % r_raw) 451 log.msg(msg) 452 self.sendStatus({'header': msg + "\n"}) 453 raise ValueError(msg) 454 return r
455 d.addCallback(_parse) 456 return d
457 458 registerSlaveCommand("bk", BK, command_version) 459 460 461 462
463 -class CVS(SourceBase):
464 """CVS-specific VC operation. In addition to the arguments handled by 465 SourceBase, this command reads the following keys: 466 467 ['cvsroot'] (required): the CVSROOT repository string 468 ['cvsmodule'] (required): the module to be retrieved 469 ['branch']: a '-r' tag or branch name to use for the checkout/update 470 ['login']: a string for use as a password to 'cvs login' 471 ['global_options']: a list of strings to use before the CVS verb 472 ['checkout_options']: a list of strings to use after checkout, 473 but before revision and branch specifiers 474 ['checkout_options']: a list of strings to use after export, 475 but before revision and branch specifiers 476 ['extra_options']: a list of strings to use after export and checkout, 477 but before revision and branch specifiers 478 """ 479 480 header = "cvs operation" 481
482 - def setup(self, args):
483 SourceBase.setup(self, args) 484 self.vcexe = getCommand("cvs") 485 self.cvsroot = args['cvsroot'] 486 self.cvsmodule = args['cvsmodule'] 487 self.global_options = args.get('global_options', []) 488 self.checkout_options = args.get('checkout_options', []) 489 self.export_options = args.get('export_options', []) 490 self.extra_options = args.get('extra_options', []) 491 self.branch = args.get('branch') 492 self.login = args.get('login') 493 self.sourcedata = "%s\n%s\n%s\n" % (self.cvsroot, self.cvsmodule, 494 self.branch)
495
496 - def sourcedirIsUpdateable(self):
497 return (not self.sourcedirIsPatched() and 498 os.path.isdir(os.path.join(self.builder.basedir, 499 self.srcdir, "CVS")))
500
501 - def start(self):
502 if self.login is not None: 503 # need to do a 'cvs login' command first 504 d = self.builder.basedir 505 command = ([self.vcexe, '-d', self.cvsroot] + self.global_options 506 + ['login']) 507 c = ShellCommand(self.builder, command, d, 508 sendRC=False, timeout=self.timeout, 509 maxTime=self.maxTime, 510 initialStdin=self.login+"\n", usePTY=False) 511 self.command = c 512 d = c.start() 513 d.addCallback(self._abandonOnFailure) 514 d.addCallback(self._didLogin) 515 return d 516 else: 517 return self._didLogin(None)
518
519 - def _didLogin(self, res):
520 # now we really start 521 return SourceBase.start(self)
522
523 - def doVCUpdate(self):
524 d = os.path.join(self.builder.basedir, self.srcdir) 525 command = [self.vcexe, '-z3'] + self.global_options + ['update', '-dP'] 526 if self.branch: 527 command += ['-r', self.branch] 528 if self.revision: 529 command += ['-D', self.revision] 530 c = ShellCommand(self.builder, command, d, 531 sendRC=False, timeout=self.timeout, 532 maxTime=self.maxTime, usePTY=False) 533 self.command = c 534 return c.start()
535
536 - def doVCFull(self):
537 d = self.builder.basedir 538 if self.mode == "export": 539 verb = "export" 540 else: 541 verb = "checkout" 542 command = ([self.vcexe, '-d', self.cvsroot, '-z3'] + 543 self.global_options + 544 [verb, '-d', self.srcdir]) 545 546 if verb == "checkout": 547 command += self.checkout_options 548 else: 549 command += self.export_options 550 command += self.extra_options 551 552 if self.branch: 553 command += ['-r', self.branch] 554 if self.revision: 555 command += ['-D', self.revision] 556 command += [self.cvsmodule] 557 558 c = ShellCommand(self.builder, command, d, 559 sendRC=False, timeout=self.timeout, 560 maxTime=self.maxTime, usePTY=False) 561 self.command = c 562 return c.start()
563
564 - def parseGotRevision(self):
565 # CVS does not have any kind of revision stamp to speak of. We return 566 # the current timestamp as a best-effort guess, but this depends upon 567 # the local system having a clock that is 568 # reasonably-well-synchronized with the repository. 569 return time.strftime("%Y-%m-%d %H:%M:%S +0000", time.gmtime())
570 571 registerSlaveCommand("cvs", CVS, command_version) 572
573 -class SVN(SourceBase):
574 """Subversion-specific VC operation. In addition to the arguments 575 handled by SourceBase, this command reads the following keys: 576 577 ['svnurl'] (required): the SVN repository string 578 ['username']: Username passed to the svn command 579 ['password']: Password passed to the svn command 580 ['keep_on_purge']: Files and directories to keep between updates 581 ['ignore_ignores']: Ignore ignores when purging changes 582 ['always_purge']: Always purge local changes after each build 583 ['depth']: Pass depth argument to subversion 1.5+ 584 """ 585 586 header = "svn operation" 587
588 - def setup(self, args):
589 SourceBase.setup(self, args) 590 self.vcexe = getCommand("svn") 591 self.svnurl = args['svnurl'] 592 self.sourcedata = "%s\n" % self.svnurl 593 self.keep_on_purge = args.get('keep_on_purge', []) 594 self.keep_on_purge.append(".buildbot-sourcedata") 595 self.ignore_ignores = args.get('ignore_ignores', True) 596 self.always_purge = args.get('always_purge', False) 597 598 self.svn_args = [] 599 if args.has_key('username'): 600 self.svn_args.extend(["--username", args['username']]) 601 if args.has_key('password'): 602 self.svn_args.extend(["--password", Obfuscated(args['password'], "XXXX")]) 603 if args.get('extra_args', None) is not None: 604 self.svn_args.extend(args['extra_args']) 605 606 if args.has_key('depth'): 607 self.svn_args.extend(["--depth",args['depth']])
608
609 - def _dovccmd(self, command, args, rootdir=None, cb=None, **kwargs):
610 if rootdir is None: 611 rootdir = os.path.join(self.builder.basedir, self.srcdir) 612 fullCmd = [self.vcexe, command, '--non-interactive', '--no-auth-cache'] 613 fullCmd.extend(self.svn_args) 614 fullCmd.extend(args) 615 c = ShellCommand(self.builder, fullCmd, rootdir, 616 environ=self.env, sendRC=False, timeout=self.timeout, 617 maxTime=self.maxTime, usePTY=False, **kwargs) 618 self.command = c 619 d = c.start() 620 if cb: 621 d.addCallback(self._abandonOnFailure) 622 d.addCallback(cb) 623 return d
624
625 - def sourcedirIsUpdateable(self):
626 return os.path.isdir(os.path.join(self.builder.basedir, 627 self.srcdir, ".svn"))
628
629 - def doVCUpdate(self):
630 if self.sourcedirIsPatched() or self.always_purge: 631 return self._purgeAndUpdate() 632 revision = self.args['revision'] or 'HEAD' 633 # update: possible for mode in ('copy', 'update') 634 return self._dovccmd('update', ['--revision', str(revision)], 635 keepStdout=True)
636
637 - def doVCFull(self):
638 revision = self.args['revision'] or 'HEAD' 639 args = ['--revision', str(revision), self.svnurl, self.srcdir] 640 if self.mode == "export": 641 command = 'export' 642 else: 643 # mode=='clobber', or copy/update on a broken workspace 644 command = 'checkout' 645 return self._dovccmd(command, args, rootdir=self.builder.basedir, 646 keepStdout=True)
647
648 - def _purgeAndUpdate(self):
649 """svn revert has several corner cases that make it unpractical. 650 651 Use the Force instead and delete everything that shows up in status.""" 652 args = ['--xml'] 653 if self.ignore_ignores: 654 args.append('--no-ignore') 655 return self._dovccmd('status', args, keepStdout=True, sendStdout=False, 656 cb=self._purgeAndUpdate2)
657
658 - def _purgeAndUpdate2(self, res):
659 """Delete everything that shown up on status.""" 660 result_xml = parseString(self.command.stdout) 661 for entry in result_xml.getElementsByTagName('entry'): 662 filename = entry.getAttribute('path') 663 if filename in self.keep_on_purge: 664 continue 665 filepath = os.path.join(self.builder.basedir, self.workdir, 666 filename) 667 self.sendStatus({'stdout': "%s\n" % filepath}) 668 if os.path.isfile(filepath): 669 os.chmod(filepath, 0700) 670 os.remove(filepath) 671 else: 672 rmdirRecursive(filepath) 673 # Now safe to update. 674 revision = self.args['revision'] or 'HEAD' 675 return self._dovccmd('update', ['--revision', str(revision)], 676 keepStdout=True)
677
678 - def getSvnVersionCommand(self):
679 """ 680 Get the (shell) command used to determine SVN revision number 681 of checked-out code 682 683 return: list of strings, passable as the command argument to ShellCommand 684 """ 685 # svn checkout operations finish with 'Checked out revision 16657.' 686 # svn update operations finish the line 'At revision 16654.' 687 # But we don't use those. Instead, run 'svnversion'. 688 svnversion_command = getCommand("svnversion") 689 # older versions of 'svnversion' (1.1.4) require the WC_PATH 690 # argument, newer ones (1.3.1) do not. 691 return [svnversion_command, "."]
692
693 - def parseGotRevision(self):
694 c = ShellCommand(self.builder, 695 self.getSvnVersionCommand(), 696 os.path.join(self.builder.basedir, self.srcdir), 697 environ=self.env, 698 sendStdout=False, sendStderr=False, sendRC=False, 699 keepStdout=True, usePTY=False) 700 d = c.start() 701 def _parse(res): 702 r_raw = c.stdout.strip() 703 # Extract revision from the version "number" string 704 r = r_raw.rstrip('MS') 705 r = r.split(':')[-1] 706 got_version = None 707 try: 708 got_version = int(r) 709 except ValueError: 710 msg =("SVN.parseGotRevision unable to parse output " 711 "of svnversion: '%s'" % r_raw) 712 log.msg(msg) 713 self.sendStatus({'header': msg + "\n"}) 714 return got_version
715 d.addCallback(_parse) 716 return d
717 718 719 registerSlaveCommand("svn", SVN, command_version) 720
721 -class Darcs(SourceBase):
722 """Darcs-specific VC operation. In addition to the arguments 723 handled by SourceBase, this command reads the following keys: 724 725 ['repourl'] (required): the Darcs repository string 726 """ 727 728 header = "darcs operation" 729
730 - def setup(self, args):
731 SourceBase.setup(self, args) 732 self.vcexe = getCommand("darcs") 733 self.repourl = args['repourl'] 734 self.sourcedata = "%s\n" % self.repourl 735 self.revision = self.args.get('revision')
736
737 - def sourcedirIsUpdateable(self):
738 # checking out a specific revision requires a full 'darcs get' 739 return (not self.revision and 740 not self.sourcedirIsPatched() and 741 os.path.isdir(os.path.join(self.builder.basedir, 742 self.srcdir, "_darcs")))
743
744 - def doVCUpdate(self):
745 assert not self.revision 746 # update: possible for mode in ('copy', 'update') 747 d = os.path.join(self.builder.basedir, self.srcdir) 748 command = [self.vcexe, 'pull', '--all', '--verbose'] 749 c = ShellCommand(self.builder, command, d, 750 sendRC=False, timeout=self.timeout, 751 maxTime=self.maxTime, usePTY=False) 752 self.command = c 753 return c.start()
754
755 - def doVCFull(self):
756 # checkout or export 757 d = self.builder.basedir 758 command = [self.vcexe, 'get', '--verbose', '--partial', 759 '--repo-name', self.srcdir] 760 if self.revision: 761 # write the context to a file 762 n = os.path.join(self.builder.basedir, ".darcs-context") 763 f = open(n, "wb") 764 f.write(self.revision) 765 f.close() 766 # tell Darcs to use that context 767 command.append('--context') 768 command.append(n) 769 command.append(self.repourl) 770 771 c = ShellCommand(self.builder, command, d, 772 sendRC=False, timeout=self.timeout, 773 maxTime=self.maxTime, usePTY=False) 774 self.command = c 775 d = c.start() 776 if self.revision: 777 d.addCallback(self.removeContextFile, n) 778 return d
779
780 - def removeContextFile(self, res, n):
781 os.unlink(n) 782 return res
783
784 - def parseGotRevision(self):
785 # we use 'darcs context' to find out what we wound up with 786 command = [self.vcexe, "changes", "--context"] 787 c = ShellCommand(self.builder, command, 788 os.path.join(self.builder.basedir, self.srcdir), 789 environ=self.env, 790 sendStdout=False, sendStderr=False, sendRC=False, 791 keepStdout=True, usePTY=False) 792 d = c.start() 793 d.addCallback(lambda res: c.stdout) 794 return d
795 796 registerSlaveCommand("darcs", Darcs, command_version) 797
798 -class Monotone(SourceBase):
799 """Monotone-specific VC operation. In addition to the arguments handled 800 by SourceBase, this command reads the following keys: 801 802 ['server_addr'] (required): the address of the server to pull from 803 ['branch'] (required): the branch the revision is on 804 ['db_path'] (required): the local database path to use 805 ['revision'] (required): the revision to check out 806 ['monotone']: (required): path to monotone executable 807 """ 808 809 header = "monotone operation" 810
811 - def setup(self, args):
812 SourceBase.setup(self, args) 813 self.server_addr = args["server_addr"] 814 self.branch = args["branch"] 815 self.db_path = args["db_path"] 816 self.revision = args["revision"] 817 self.monotone = args["monotone"] 818 self._made_fulls = False 819 self._pull_timeout = args["timeout"]
820
821 - def _makefulls(self):
822 if not self._made_fulls: 823 basedir = self.builder.basedir 824 self.full_db_path = os.path.join(basedir, self.db_path) 825 self.full_srcdir = os.path.join(basedir, self.srcdir) 826 self._made_fulls = True
827
828 - def sourcedirIsUpdateable(self):
829 self._makefulls() 830 return (not self.sourcedirIsPatched() and 831 os.path.isfile(self.full_db_path) and 832 os.path.isdir(os.path.join(self.full_srcdir, "MT")))
833
834 - def doVCUpdate(self):
835 return self._withFreshDb(self._doUpdate)
836
837 - def _doUpdate(self):
838 # update: possible for mode in ('copy', 'update') 839 command = [self.monotone, "update", 840 "-r", self.revision, 841 "-b", self.branch] 842 c = ShellCommand(self.builder, command, self.full_srcdir, 843 sendRC=False, timeout=self.timeout, 844 maxTime=self.maxTime, usePTY=False) 845 self.command = c 846 return c.start()
847
848 - def doVCFull(self):
849 return self._withFreshDb(self._doFull)
850
851 - def _doFull(self):
852 command = [self.monotone, "--db=" + self.full_db_path, 853 "checkout", 854 "-r", self.revision, 855 "-b", self.branch, 856 self.full_srcdir] 857 c = ShellCommand(self.builder, command, self.builder.basedir, 858 sendRC=False, timeout=self.timeout, 859 maxTime=self.maxTime, usePTY=False) 860 self.command = c 861 return c.start()
862
863 - def _withFreshDb(self, callback):
864 self._makefulls() 865 # first ensure the db exists and is usable 866 if os.path.isfile(self.full_db_path): 867 # already exists, so run 'db migrate' in case monotone has been 868 # upgraded under us 869 command = [self.monotone, "db", "migrate", 870 "--db=" + self.full_db_path] 871 else: 872 # We'll be doing an initial pull, so up the timeout to 3 hours to 873 # make sure it will have time to complete. 874 self._pull_timeout = max(self._pull_timeout, 3 * 60 * 60) 875 self.sendStatus({"header": "creating database %s\n" 876 % (self.full_db_path,)}) 877 command = [self.monotone, "db", "init", 878 "--db=" + self.full_db_path] 879 c = ShellCommand(self.builder, command, self.builder.basedir, 880 sendRC=False, timeout=self.timeout, 881 maxTime=self.maxTime, usePTY=False) 882 self.command = c 883 d = c.start() 884 d.addCallback(self._abandonOnFailure) 885 d.addCallback(self._didDbInit) 886 d.addCallback(self._didPull, callback) 887 return d
888
889 - def _didDbInit(self, res):
890 command = [self.monotone, "--db=" + self.full_db_path, 891 "pull", "--ticker=dot", self.server_addr, self.branch] 892 c = ShellCommand(self.builder, command, self.builder.basedir, 893 sendRC=False, timeout=self._pull_timeout, 894 maxTime=self.maxTime, usePTY=False) 895 self.sendStatus({"header": "pulling %s from %s\n" 896 % (self.branch, self.server_addr)}) 897 self.command = c 898 return c.start()
899
900 - def _didPull(self, res, callback):
901 return callback()
902 903 registerSlaveCommand("monotone", Monotone, command_version) 904 905
906 -class Git(SourceBase):
907 """Git specific VC operation. In addition to the arguments 908 handled by SourceBase, this command reads the following keys: 909 910 ['repourl'] (required): the upstream GIT repository string 911 ['branch'] (optional): which version (i.e. branch or tag) to 912 retrieve. Default: "master". 913 ['submodules'] (optional): whether to initialize and update 914 submodules. Default: False. 915 ['ignore_ignores']: ignore ignores when purging changes. 916 """ 917 918 header = "git operation" 919
920 - def setup(self, args):
921 SourceBase.setup(self, args) 922 self.vcexe = getCommand("git") 923 self.repourl = args['repourl'] 924 self.branch = args.get('branch') 925 if not self.branch: 926 self.branch = "master" 927 self.sourcedata = "%s %s\n" % (self.repourl, self.branch) 928 self.submodules = args.get('submodules') 929 self.ignore_ignores = args.get('ignore_ignores', True)
930
931 - def _fullSrcdir(self):
932 return os.path.join(self.builder.basedir, self.srcdir)
933
934 - def _commitSpec(self):
935 if self.revision: 936 return self.revision 937 return self.branch
938
939 - def sourcedirIsUpdateable(self):
940 return os.path.isdir(os.path.join(self._fullSrcdir(), ".git"))
941
942 - def _dovccmd(self, command, cb=None, **kwargs):
943 c = ShellCommand(self.builder, [self.vcexe] + command, self._fullSrcdir(), 944 sendRC=False, timeout=self.timeout, 945 maxTime=self.maxTime, usePTY=False, **kwargs) 946 self.command = c 947 d = c.start() 948 if cb: 949 d.addCallback(self._abandonOnFailure) 950 d.addCallback(cb) 951 return d
952 953 # If the repourl matches the sourcedata file, then 954 # we can say that the sourcedata matches. We can 955 # ignore branch changes, since Git can work with 956 # many branches fetched, and we deal with it properly 957 # in doVCUpdate.
958 - def sourcedataMatches(self):
959 try: 960 olddata = self.readSourcedata() 961 if not olddata.startswith(self.repourl+' '): 962 return False 963 except IOError: 964 return False 965 return True
966
967 - def _cleanSubmodules(self, res):
968 command = ['submodule', 'foreach', 'git', 'clean', '-d', '-f'] 969 if self.ignore_ignores: 970 command.append('-x') 971 return self._dovccmd(command)
972
973 - def _updateSubmodules(self, res):
974 return self._dovccmd(['submodule', 'update'], self._cleanSubmodules)
975
976 - def _initSubmodules(self, res):
977 if self.submodules: 978 return self._dovccmd(['submodule', 'init'], self._updateSubmodules) 979 else: 980 return defer.succeed(0)
981
982 - def _didHeadCheckout(self, res):
983 # Rename branch, so that the repo will have the expected branch name 984 # For further information about this, see the commit message 985 command = ['branch', '-M', self.branch] 986 return self._dovccmd(command, self._initSubmodules)
987
988 - def _didFetch(self, res):
989 if self.revision: 990 head = self.revision 991 else: 992 head = 'FETCH_HEAD' 993 994 # That is not sufficient. git will leave unversioned files and empty 995 # directories. Clean them up manually in _didReset. 996 command = ['reset', '--hard', head] 997 return self._dovccmd(command, self._didHeadCheckout)
998 999 # Update first runs "git clean", removing local changes, 1000 # if the branch to be checked out has changed. This, combined 1001 # with the later "git reset" equates clobbering the repo, 1002 # but it's much more efficient.
1003 - def doVCUpdate(self):
1004 try: 1005 # Check to see if our branch has changed 1006 diffbranch = self.sourcedata != self.readSourcedata() 1007 except IOError: 1008 diffbranch = False 1009 if diffbranch: 1010 command = ['clean', '-f', '-d'] 1011 if self.ignore_ignores: 1012 command.append('-x') 1013 return self._dovccmd(command, self._didClean) 1014 return self._didClean(None)
1015
1016 - def _doFetch(self, dummy):
1017 # The plus will make sure the repo is moved to the branch's 1018 # head even if it is not a simple "fast-forward" 1019 command = ['fetch', '-t', self.repourl, '+%s' % self.branch] 1020 self.sendStatus({"header": "fetching branch %s from %s\n" 1021 % (self.branch, self.repourl)}) 1022 return self._dovccmd(command, self._didFetch)
1023
1024 - def _didClean(self, dummy):
1025 # After a clean, try to use the given revision if we have one. 1026 if self.revision: 1027 # We know what revision we want. See if we have it. 1028 d = self._dovccmd(['reset', '--hard', self.revision], 1029 self._initSubmodules) 1030 # If we are unable to reset to the specified version, we 1031 # must do a fetch first and retry. 1032 d.addErrback(self._doFetch) 1033 return d 1034 else: 1035 # No known revision, go grab the latest. 1036 return self._doFetch(None)
1037
1038 - def _didInit(self, res):
1039 return self.doVCUpdate()
1040
1041 - def doVCFull(self):
1042 # If they didn't ask for a specific revision, we can get away with a 1043 # shallow clone. 1044 if not self.args.get('revision') and self.args.get('shallow'): 1045 cmd = [self.vcexe, 'clone', '--depth', '1', self.repourl, 1046 self._fullSrcdir()] 1047 c = ShellCommand(self.builder, cmd, self.builder.basedir, 1048 sendRC=False, timeout=self.timeout, 1049 maxTime=self.maxTime, usePTY=False) 1050 self.command = c 1051 cmdexec = c.start() 1052 cmdexec.addCallback(self._didInit) 1053 return cmdexec 1054 else: 1055 os.makedirs(self._fullSrcdir()) 1056 return self._dovccmd(['init'], self._didInit)
1057
1058 - def parseGotRevision(self):
1059 command = ['rev-parse', 'HEAD'] 1060 def _parse(res): 1061 hash = self.command.stdout.strip() 1062 if len(hash) != 40: 1063 return None 1064 return hash
1065 return self._dovccmd(command, _parse, keepStdout=True)
1066 1067 registerSlaveCommand("git", Git, command_version) 1068
1069 -class Arch(SourceBase):
1070 """Arch-specific (tla-specific) VC operation. In addition to the 1071 arguments handled by SourceBase, this command reads the following keys: 1072 1073 ['url'] (required): the repository string 1074 ['version'] (required): which version (i.e. branch) to retrieve 1075 ['revision'] (optional): the 'patch-NN' argument to check out 1076 ['archive']: the archive name to use. If None, use the archive's default 1077 ['build-config']: if present, give to 'tla build-config' after checkout 1078 """ 1079 1080 header = "arch operation" 1081 buildconfig = None 1082
1083 - def setup(self, args):
1084 SourceBase.setup(self, args) 1085 self.vcexe = getCommand("tla") 1086 self.archive = args.get('archive') 1087 self.url = args['url'] 1088 self.version = args['version'] 1089 self.revision = args.get('revision') 1090 self.buildconfig = args.get('build-config') 1091 self.sourcedata = "%s\n%s\n%s\n" % (self.url, self.version, 1092 self.buildconfig)
1093
1094 - def sourcedirIsUpdateable(self):
1095 # Arch cannot roll a directory backwards, so if they ask for a 1096 # specific revision, clobber the directory. Technically this 1097 # could be limited to the cases where the requested revision is 1098 # later than our current one, but it's too hard to extract the 1099 # current revision from the tree. 1100 return (not self.revision and 1101 not self.sourcedirIsPatched() and 1102 os.path.isdir(os.path.join(self.builder.basedir, 1103 self.srcdir, "{arch}")))
1104
1105 - def doVCUpdate(self):
1106 # update: possible for mode in ('copy', 'update') 1107 d = os.path.join(self.builder.basedir, self.srcdir) 1108 command = [self.vcexe, 'replay'] 1109 if self.revision: 1110 command.append(self.revision) 1111 c = ShellCommand(self.builder, command, d, 1112 sendRC=False, timeout=self.timeout, 1113 maxTime=self.maxTime, usePTY=False) 1114 self.command = c 1115 return c.start()
1116
1117 - def doVCFull(self):
1118 # to do a checkout, we must first "register" the archive by giving 1119 # the URL to tla, which will go to the repository at that URL and 1120 # figure out the archive name. tla will tell you the archive name 1121 # when it is done, and all further actions must refer to this name. 1122 1123 command = [self.vcexe, 'register-archive', '--force', self.url] 1124 c = ShellCommand(self.builder, command, self.builder.basedir, 1125 sendRC=False, keepStdout=True, timeout=self.timeout, 1126 maxTime=self.maxTime, usePTY=False) 1127 self.command = c 1128 d = c.start() 1129 d.addCallback(self._abandonOnFailure) 1130 d.addCallback(self._didRegister, c) 1131 return d
1132
1133 - def _didRegister(self, res, c):
1134 # find out what tla thinks the archive name is. If the user told us 1135 # to use something specific, make sure it matches. 1136 r = re.search(r'Registering archive: (\S+)\s*$', c.stdout) 1137 if r: 1138 msg = "tla reports archive name is '%s'" % r.group(1) 1139 log.msg(msg) 1140 self.builder.sendUpdate({'header': msg+"\n"}) 1141 if self.archive and r.group(1) != self.archive: 1142 msg = (" mismatch, we wanted an archive named '%s'" 1143 % self.archive) 1144 log.msg(msg) 1145 self.builder.sendUpdate({'header': msg+"\n"}) 1146 raise AbandonChain(-1) 1147 self.archive = r.group(1) 1148 assert self.archive, "need archive name to continue" 1149 return self._doGet()
1150
1151 - def _doGet(self):
1152 ver = self.version 1153 if self.revision: 1154 ver += "--%s" % self.revision 1155 command = [self.vcexe, 'get', '--archive', self.archive, 1156 '--no-pristine', 1157 ver, self.srcdir] 1158 c = ShellCommand(self.builder, command, self.builder.basedir, 1159 sendRC=False, timeout=self.timeout, 1160 maxTime=self.maxTime, usePTY=False) 1161 self.command = c 1162 d = c.start() 1163 d.addCallback(self._abandonOnFailure) 1164 if self.buildconfig: 1165 d.addCallback(self._didGet) 1166 return d
1167
1168 - def _didGet(self, res):
1169 d = os.path.join(self.builder.basedir, self.srcdir) 1170 command = [self.vcexe, 'build-config', self.buildconfig] 1171 c = ShellCommand(self.builder, command, d, 1172 sendRC=False, timeout=self.timeout, 1173 maxTime=self.maxTime, usePTY=False) 1174 self.command = c 1175 d = c.start() 1176 d.addCallback(self._abandonOnFailure) 1177 return d
1178
1179 - def parseGotRevision(self):
1180 # using code from tryclient.TlaExtractor 1181 # 'tla logs --full' gives us ARCHIVE/BRANCH--REVISION 1182 # 'tla logs' gives us REVISION 1183 command = [self.vcexe, "logs", "--full", "--reverse"] 1184 c = ShellCommand(self.builder, command, 1185 os.path.join(self.builder.basedir, self.srcdir), 1186 environ=self.env, 1187 sendStdout=False, sendStderr=False, sendRC=False, 1188 keepStdout=True, usePTY=False) 1189 d = c.start() 1190 def _parse(res): 1191 tid = c.stdout.split("\n")[0].strip() 1192 slash = tid.index("/") 1193 dd = tid.rindex("--") 1194 #branch = tid[slash+1:dd] 1195 baserev = tid[dd+2:] 1196 return baserev
1197 d.addCallback(_parse) 1198 return d
1199 1200 registerSlaveCommand("arch", Arch, command_version) 1201
1202 -class Bazaar(Arch):
1203 """Bazaar (/usr/bin/baz) is an alternative client for Arch repositories. 1204 It is mostly option-compatible, but archive registration is different 1205 enough to warrant a separate Command. 1206 1207 ['archive'] (required): the name of the archive being used 1208 """ 1209
1210 - def setup(self, args):
1211 Arch.setup(self, args) 1212 self.vcexe = getCommand("baz") 1213 # baz doesn't emit the repository name after registration (and 1214 # grepping through the output of 'baz archives' is too hard), so we 1215 # require that the buildmaster configuration to provide both the 1216 # archive name and the URL. 1217 self.archive = args['archive'] # required for Baz 1218 self.sourcedata = "%s\n%s\n%s\n" % (self.url, self.version, 1219 self.buildconfig)
1220 1221 # in _didRegister, the regexp won't match, so we'll stick with the name 1222 # in self.archive 1223
1224 - def _doGet(self):
1225 # baz prefers ARCHIVE/VERSION. This will work even if 1226 # my-default-archive is not set. 1227 ver = self.archive + "/" + self.version 1228 if self.revision: 1229 ver += "--%s" % self.revision 1230 command = [self.vcexe, 'get', '--no-pristine', 1231 ver, self.srcdir] 1232 c = ShellCommand(self.builder, command, self.builder.basedir, 1233 sendRC=False, timeout=self.timeout, 1234 maxTime=self.maxTime, usePTY=False) 1235 self.command = c 1236 d = c.start() 1237 d.addCallback(self._abandonOnFailure) 1238 if self.buildconfig: 1239 d.addCallback(self._didGet) 1240 return d
1241
1242 - def parseGotRevision(self):
1243 # using code from tryclient.BazExtractor 1244 command = [self.vcexe, "tree-id"] 1245 c = ShellCommand(self.builder, command, 1246 os.path.join(self.builder.basedir, self.srcdir), 1247 environ=self.env, 1248 sendStdout=False, sendStderr=False, sendRC=False, 1249 keepStdout=True, usePTY=False) 1250 d = c.start() 1251 def _parse(res): 1252 tid = c.stdout.strip() 1253 slash = tid.index("/") 1254 dd = tid.rindex("--") 1255 #branch = tid[slash+1:dd] 1256 baserev = tid[dd+2:] 1257 return baserev
1258 d.addCallback(_parse) 1259 return d
1260 1261 registerSlaveCommand("bazaar", Bazaar, command_version) 1262 1263
1264 -class Bzr(SourceBase):
1265 """bzr-specific VC operation. In addition to the arguments 1266 handled by SourceBase, this command reads the following keys: 1267 1268 ['repourl'] (required): the Bzr repository string 1269 """ 1270 1271 header = "bzr operation" 1272
1273 - def setup(self, args):
1274 SourceBase.setup(self, args) 1275 self.vcexe = getCommand("bzr") 1276 self.repourl = args['repourl'] 1277 self.sourcedata = "%s\n" % self.repourl 1278 self.revision = self.args.get('revision') 1279 self.forceSharedRepo = args.get('forceSharedRepo')
1280
1281 - def sourcedirIsUpdateable(self):
1282 # checking out a specific revision requires a full 'bzr checkout' 1283 return (not self.revision and 1284 not self.sourcedirIsPatched() and 1285 os.path.isdir(os.path.join(self.builder.basedir, 1286 self.srcdir, ".bzr")))
1287
1288 - def start(self):
1289 def cont(res): 1290 # Continue with start() method in superclass. 1291 return SourceBase.start(self)
1292 1293 if self.forceSharedRepo: 1294 d = self.doForceSharedRepo(); 1295 d.addCallback(cont) 1296 return d 1297 else: 1298 return cont(None)
1299
1300 - def doVCUpdate(self):
1301 assert not self.revision 1302 # update: possible for mode in ('copy', 'update') 1303 srcdir = os.path.join(self.builder.basedir, self.srcdir) 1304 command = [self.vcexe, 'update'] 1305 c = ShellCommand(self.builder, command, srcdir, 1306 sendRC=False, timeout=self.timeout, 1307 maxTime=self.maxTime, usePTY=False) 1308 self.command = c 1309 return c.start()
1310
1311 - def doVCFull(self):
1312 # checkout or export 1313 d = self.builder.basedir 1314 if self.mode == "export": 1315 # exporting in bzr requires a separate directory 1316 return self.doVCExport() 1317 # originally I added --lightweight here, but then 'bzr revno' is 1318 # wrong. The revno reported in 'bzr version-info' is correct, 1319 # however. Maybe this is a bzr bug? 1320 # 1321 # In addition, you cannot perform a 'bzr update' on a repo pulled 1322 # from an HTTP repository that used 'bzr checkout --lightweight'. You 1323 # get a "ERROR: Cannot lock: transport is read only" when you try. 1324 # 1325 # So I won't bother using --lightweight for now. 1326 1327 command = [self.vcexe, 'checkout'] 1328 if self.revision: 1329 command.append('--revision') 1330 command.append(str(self.revision)) 1331 command.append(self.repourl) 1332 command.append(self.srcdir) 1333 1334 c = ShellCommand(self.builder, command, d, 1335 sendRC=False, timeout=self.timeout, 1336 maxTime=self.maxTime, usePTY=False) 1337 self.command = c 1338 d = c.start() 1339 return d
1340
1341 - def doVCExport(self):
1342 tmpdir = os.path.join(self.builder.basedir, "export-temp") 1343 srcdir = os.path.join(self.builder.basedir, self.srcdir) 1344 command = [self.vcexe, 'checkout', '--lightweight'] 1345 if self.revision: 1346 command.append('--revision') 1347 command.append(str(self.revision)) 1348 command.append(self.repourl) 1349 command.append(tmpdir) 1350 c = ShellCommand(self.builder, command, self.builder.basedir, 1351 sendRC=False, timeout=self.timeout, 1352 maxTime=self.maxTime, usePTY=False) 1353 self.command = c 1354 d = c.start() 1355 def _export(res): 1356 command = [self.vcexe, 'export', srcdir] 1357 c = ShellCommand(self.builder, command, tmpdir, 1358 sendRC=False, timeout=self.timeout, 1359 maxTime=self.maxTime, usePTY=False) 1360 self.command = c 1361 return c.start()
1362 d.addCallback(_export) 1363 return d 1364
1365 - def doForceSharedRepo(self):
1366 # Don't send stderr. When there is no shared repo, this might confuse 1367 # users, as they will see a bzr error message. But having no shared 1368 # repo is not an error, just an indication that we need to make one. 1369 c = ShellCommand(self.builder, [self.vcexe, 'info', '.'], 1370 self.builder.basedir, 1371 sendStderr=False, sendRC=False, usePTY=False) 1372 d = c.start() 1373 def afterCheckSharedRepo(res): 1374 if type(res) is int and res != 0: 1375 log.msg("No shared repo found, creating it") 1376 # bzr info fails, try to create shared repo. 1377 c = ShellCommand(self.builder, [self.vcexe, 'init-repo', '.'], 1378 self.builder.basedir, 1379 sendRC=False, usePTY=False) 1380 self.command = c 1381 return c.start() 1382 else: 1383 return defer.succeed(res)
1384 d.addCallback(afterCheckSharedRepo) 1385 return d 1386
1387 - def get_revision_number(self, out):
1388 # it feels like 'bzr revno' sometimes gives different results than 1389 # the 'revno:' line from 'bzr version-info', and the one from 1390 # version-info is more likely to be correct. 1391 for line in out.split("\n"): 1392 colon = line.find(":") 1393 if colon != -1: 1394 key, value = line[:colon], line[colon+2:] 1395 if key == "revno": 1396 return int(value) 1397 raise ValueError("unable to find revno: in bzr output: '%s'" % out)
1398
1399 - def parseGotRevision(self):
1400 command = [self.vcexe, "version-info"] 1401 c = ShellCommand(self.builder, command, 1402 os.path.join(self.builder.basedir, self.srcdir), 1403 environ=self.env, 1404 sendStdout=False, sendStderr=False, sendRC=False, 1405 keepStdout=True, usePTY=False) 1406 d = c.start() 1407 def _parse(res): 1408 try: 1409 return self.get_revision_number(c.stdout) 1410 except ValueError: 1411 msg =("Bzr.parseGotRevision unable to parse output " 1412 "of bzr version-info: '%s'" % c.stdout.strip()) 1413 log.msg(msg) 1414 self.sendStatus({'header': msg + "\n"}) 1415 return None
1416 d.addCallback(_parse) 1417 return d 1418 1419 registerSlaveCommand("bzr", Bzr, command_version) 1420
1421 -class Mercurial(SourceBase):
1422 """Mercurial specific VC operation. In addition to the arguments 1423 handled by SourceBase, this command reads the following keys: 1424 1425 ['repourl'] (required): the Mercurial repository string 1426 ['clobberOnBranchChange']: Document me. See ticket #462. 1427 """ 1428 1429 header = "mercurial operation" 1430
1431 - def setup(self, args):
1432 SourceBase.setup(self, args) 1433 self.vcexe = getCommand("hg") 1434 self.repourl = args['repourl'] 1435 self.clobberOnBranchChange = args.get('clobberOnBranchChange', True) 1436 self.sourcedata = "%s\n" % self.repourl 1437 self.branchType = args.get('branchType', 'dirname') 1438 self.stdout = "" 1439 self.stderr = "" 1440 self.clobbercount = 0 # n times we've clobbered
1441
1442 - def sourcedirIsUpdateable(self):
1443 return os.path.isdir(os.path.join(self.builder.basedir, 1444 self.srcdir, ".hg"))
1445
1446 - def doVCUpdate(self):
1447 d = os.path.join(self.builder.basedir, self.srcdir) 1448 command = [self.vcexe, 'pull', '--verbose', self.repourl] 1449 c = ShellCommand(self.builder, command, d, 1450 sendRC=False, timeout=self.timeout, 1451 maxTime=self.maxTime, keepStdout=True, usePTY=False) 1452 self.command = c 1453 d = c.start() 1454 d.addCallback(self._handleEmptyUpdate) 1455 d.addCallback(self._update) 1456 return d
1457
1458 - def _handleEmptyUpdate(self, res):
1459 if type(res) is int and res == 1: 1460 if self.command.stdout.find("no changes found") != -1: 1461 # 'hg pull', when it doesn't have anything to do, exits with 1462 # rc=1, and there appears to be no way to shut this off. It 1463 # emits a distinctive message to stdout, though. So catch 1464 # this and pretend that it completed successfully. 1465 return 0 1466 return res
1467
1468 - def doVCFull(self):
1469 d = os.path.join(self.builder.basedir, self.srcdir) 1470 command = [self.vcexe, 'clone', '--verbose', '--noupdate'] 1471 1472 # if got revision, clobbering and in dirname, only clone to specific revision 1473 # (otherwise, do full clone to re-use .hg dir for subsequent builds) 1474 if self.args.get('revision') and self.mode == 'clobber' and self.branchType == 'dirname': 1475 command.extend(['--rev', self.args.get('revision')]) 1476 command.extend([self.repourl, d]) 1477 1478 c = ShellCommand(self.builder, command, self.builder.basedir, 1479 sendRC=False, timeout=self.timeout, 1480 maxTime=self.maxTime, usePTY=False) 1481 self.command = c 1482 cmd1 = c.start() 1483 cmd1.addCallback(self._update) 1484 return cmd1
1485
1486 - def _clobber(self, dummy, dirname):
1487 self.clobbercount += 1 1488 1489 if self.clobbercount > 3: 1490 raise Exception, "Too many clobber attempts. Aborting step" 1491 1492 def _vcfull(res): 1493 return self.doVCFull()
1494 1495 c = self.doClobber(dummy, dirname) 1496 c.addCallback(_vcfull) 1497 1498 return c
1499
1500 - def _purge(self, dummy, dirname):
1501 d = os.path.join(self.builder.basedir, self.srcdir) 1502 purge = [self.vcexe, 'purge', '--all'] 1503 purgeCmd = ShellCommand(self.builder, purge, d, 1504 sendStdout=False, sendStderr=False, 1505 keepStdout=True, keepStderr=True, usePTY=False) 1506 1507 def _clobber(res): 1508 if res != 0: 1509 # purge failed, we need to switch to a classic clobber 1510 msg = "'hg purge' failed: %s\n%s. Clobbering." % (purgeCmd.stdout, purgeCmd.stderr) 1511 self.sendStatus({'header': msg + "\n"}) 1512 log.msg(msg) 1513 1514 return self._clobber(dummy, dirname) 1515 1516 # Purge was a success, then we need to update 1517 return self._update2(res)
1518 1519 p = purgeCmd.start() 1520 p.addCallback(_clobber) 1521 return p 1522
1523 - def _update(self, res):
1524 if res != 0: 1525 return res 1526 1527 # compare current branch to update 1528 self.update_branch = self.args.get('branch', 'default') 1529 1530 d = os.path.join(self.builder.basedir, self.srcdir) 1531 parentscmd = [self.vcexe, 'identify', '--num', '--branch'] 1532 cmd = ShellCommand(self.builder, parentscmd, d, 1533 sendStdout=False, sendStderr=False, 1534 keepStdout=True, keepStderr=True, usePTY=False) 1535 1536 self.clobber = None 1537 1538 def _parseIdentify(res): 1539 if res != 0: 1540 msg = "'hg identify' failed: %s\n%s" % (cmd.stdout, cmd.stderr) 1541 self.sendStatus({'header': msg + "\n"}) 1542 log.msg(msg) 1543 return res 1544 1545 log.msg('Output: %s' % cmd.stdout) 1546 1547 match = re.search(r'^(.+) (.+)$', cmd.stdout) 1548 assert match 1549 1550 rev = match.group(1) 1551 current_branch = match.group(2) 1552 1553 if rev == '-1': 1554 msg = "Fresh hg repo, don't worry about in-repo branch name" 1555 log.msg(msg) 1556 1557 elif self.sourcedirIsPatched(): 1558 self.clobber = self._purge 1559 1560 elif self.update_branch != current_branch: 1561 msg = "Working dir is on in-repo branch '%s' and build needs '%s'." % (current_branch, self.update_branch) 1562 if self.clobberOnBranchChange: 1563 msg += ' Cloberring.' 1564 else: 1565 msg += ' Updating.' 1566 1567 self.sendStatus({'header': msg + "\n"}) 1568 log.msg(msg) 1569 1570 # Clobbers only if clobberOnBranchChange is set 1571 if self.clobberOnBranchChange: 1572 self.clobber = self._purge 1573 1574 else: 1575 msg = "Working dir on same in-repo branch as build (%s)." % (current_branch) 1576 log.msg(msg) 1577 1578 return 0
1579 1580 def _checkRepoURL(res): 1581 parentscmd = [self.vcexe, 'paths', 'default'] 1582 cmd2 = ShellCommand(self.builder, parentscmd, d, 1583 sendStdout=False, sendStderr=False, 1584 keepStdout=True, keepStderr=True, usePTY=False) 1585 1586 def _parseRepoURL(res): 1587 if res == 1: 1588 if "not found!" == cmd2.stderr.strip(): 1589 msg = "hg default path not set. Not checking repo url for clobber test" 1590 log.msg(msg) 1591 return 0 1592 else: 1593 msg = "'hg paths default' failed: %s\n%s" % (cmd2.stdout, cmd2.stderr) 1594 log.msg(msg) 1595 return 1 1596 1597 oldurl = cmd2.stdout.strip() 1598 1599 log.msg("Repo cloned from: '%s'" % oldurl) 1600 1601 if runtime.platformType == 'win32': 1602 oldurl = oldurl.lower().replace('\\', '/') 1603 repourl = self.repourl.lower().replace('\\', '/') 1604 else: 1605 repourl = self.repourl 1606 1607 if repourl.startswith('file://'): 1608 repourl = repourl.split('file://')[1] 1609 if oldurl.startswith('file://'): 1610 oldurl = oldurl.split('file://')[1] 1611 1612 oldurl = remove_userpassword(oldurl) 1613 repourl = remove_userpassword(repourl) 1614 1615 if oldurl.rstrip('/') != repourl.rstrip('/'): 1616 self.clobber = self._clobber 1617 msg = "RepoURL changed from '%s' in wc to '%s' in update. Clobbering" % (oldurl, repourl) 1618 log.msg(msg) 1619 1620 return 0 1621 1622 c = cmd2.start() 1623 c.addCallback(_parseRepoURL) 1624 return c 1625 1626 def _maybeClobber(res): 1627 if self.clobber: 1628 msg = "Clobber flag set. Doing clobbering" 1629 log.msg(msg) 1630 1631 def _vcfull(res): 1632 return self.doVCFull() 1633 1634 return self.clobber(None, self.srcdir) 1635 1636 return 0 1637 1638 c = cmd.start() 1639 c.addCallback(_parseIdentify) 1640 c.addCallback(_checkRepoURL) 1641 c.addCallback(_maybeClobber) 1642 c.addCallback(self._update2) 1643 return c 1644
1645 - def _update2(self, res):
1646 d = os.path.join(self.builder.basedir, self.srcdir) 1647 1648 updatecmd=[self.vcexe, 'update', '--clean', '--repository', d] 1649 if self.args.get('revision'): 1650 updatecmd.extend(['--rev', self.args['revision']]) 1651 else: 1652 updatecmd.extend(['--rev', self.args.get('branch', 'default')]) 1653 self.command = ShellCommand(self.builder, updatecmd, 1654 self.builder.basedir, sendRC=False, 1655 timeout=self.timeout, maxTime=self.maxTime, usePTY=False) 1656 return self.command.start()
1657
1658 - def parseGotRevision(self):
1659 # we use 'hg identify' to find out what we wound up with 1660 command = [self.vcexe, "identify", "--id", "--debug"] # get full rev id 1661 c = ShellCommand(self.builder, command, 1662 os.path.join(self.builder.basedir, self.srcdir), 1663 environ=self.env, 1664 sendStdout=False, sendStderr=False, sendRC=False, 1665 keepStdout=True, usePTY=False) 1666 d = c.start() 1667 def _parse(res): 1668 m = re.search(r'^(\w+)', c.stdout) 1669 return m.group(1)
1670 d.addCallback(_parse) 1671 return d 1672 1673 registerSlaveCommand("hg", Mercurial, command_version) 1674 1675
1676 -class P4Base(SourceBase):
1677 """Base class for P4 source-updaters 1678 1679 ['p4port'] (required): host:port for server to access 1680 ['p4user'] (optional): user to use for access 1681 ['p4passwd'] (optional): passwd to try for the user 1682 ['p4client'] (optional): client spec to use 1683 """
1684 - def setup(self, args):
1685 SourceBase.setup(self, args) 1686 self.p4port = args['p4port'] 1687 self.p4client = args['p4client'] 1688 self.p4user = args['p4user'] 1689 self.p4passwd = args['p4passwd']
1690
1691 - def parseGotRevision(self):
1692 # Executes a p4 command that will give us the latest changelist number 1693 # of any file under the current (or default) client: 1694 command = ['p4'] 1695 if self.p4port: 1696 command.extend(['-p', self.p4port]) 1697 if self.p4user: 1698 command.extend(['-u', self.p4user]) 1699 if self.p4passwd: 1700 command.extend(['-P', Obfuscated(self.p4passwd, "XXXXXXXX")]) 1701 if self.p4client: 1702 command.extend(['-c', self.p4client]) 1703 # add '-s submitted' for bug #626 1704 command.extend(['changes', '-s', 'submitted', '-m', '1', '#have']) 1705 c = ShellCommand(self.builder, command, self.builder.basedir, 1706 environ=self.env, timeout=self.timeout, 1707 maxTime=self.maxTime, sendStdout=True, 1708 sendStderr=False, sendRC=False, keepStdout=True, 1709 usePTY=False) 1710 self.command = c 1711 d = c.start() 1712 1713 def _parse(res): 1714 # 'p4 -c clien-name change -m 1 "#have"' will produce an output like: 1715 # "Change 28147 on 2008/04/07 by p4user@hostname..." 1716 # The number after "Change" is the one we want. 1717 m = re.match('Change\s+(\d+)\s+', c.stdout) 1718 if m: 1719 return m.group(1) 1720 return None
1721 d.addCallback(_parse) 1722 return d
1723 1724
1725 -class P4(P4Base):
1726 """A P4 source-updater. 1727 1728 ['p4port'] (required): host:port for server to access 1729 ['p4user'] (optional): user to use for access 1730 ['p4passwd'] (optional): passwd to try for the user 1731 ['p4client'] (optional): client spec to use 1732 ['p4extra_views'] (optional): additional client views to use 1733 """ 1734 1735 header = "p4" 1736
1737 - def setup(self, args):
1738 P4Base.setup(self, args) 1739 self.p4base = args['p4base'] 1740 self.p4extra_views = args['p4extra_views'] 1741 self.p4mode = args['mode'] 1742 self.p4branch = args['branch'] 1743 1744 self.sourcedata = str([ 1745 # Perforce server. 1746 self.p4port, 1747 1748 # Client spec. 1749 self.p4client, 1750 1751 # Depot side of view spec. 1752 self.p4base, 1753 self.p4branch, 1754 self.p4extra_views, 1755 1756 # Local side of view spec (srcdir is made from these). 1757 self.builder.basedir, 1758 self.mode, 1759 self.workdir 1760 ])
1761 1762
1763 - def sourcedirIsUpdateable(self):
1764 # We assume our client spec is still around. 1765 # We just say we aren't updateable if the dir doesn't exist so we 1766 # don't get ENOENT checking the sourcedata. 1767 return (not self.sourcedirIsPatched() and 1768 os.path.isdir(os.path.join(self.builder.basedir, 1769 self.srcdir)))
1770
1771 - def doVCUpdate(self):
1772 return self._doP4Sync(force=False)
1773
1774 - def _doP4Sync(self, force):
1775 command = ['p4'] 1776 1777 if self.p4port: 1778 command.extend(['-p', self.p4port]) 1779 if self.p4user: 1780 command.extend(['-u', self.p4user]) 1781 if self.p4passwd: 1782 command.extend(['-P', Obfuscated(self.p4passwd, "XXXXXXXX")]) 1783 if self.p4client: 1784 command.extend(['-c', self.p4client]) 1785 command.extend(['sync']) 1786 if force: 1787 command.extend(['-f']) 1788 if self.revision: 1789 command.extend(['@' + str(self.revision)]) 1790 env = {} 1791 c = ShellCommand(self.builder, command, self.builder.basedir, 1792 environ=env, sendRC=False, timeout=self.timeout, 1793 maxTime=self.maxTime, keepStdout=True, usePTY=False) 1794 self.command = c 1795 d = c.start() 1796 d.addCallback(self._abandonOnFailure) 1797 return d
1798 1799
1800 - def doVCFull(self):
1801 env = {} 1802 command = ['p4'] 1803 client_spec = '' 1804 client_spec += "Client: %s\n\n" % self.p4client 1805 client_spec += "Owner: %s\n\n" % self.p4user 1806 client_spec += "Description:\n\tCreated by %s\n\n" % self.p4user 1807 client_spec += "Root:\t%s\n\n" % self.builder.basedir 1808 client_spec += "Options:\tallwrite rmdir\n\n" 1809 client_spec += "LineEnd:\tlocal\n\n" 1810 1811 # Setup a view 1812 client_spec += "View:\n\t%s" % (self.p4base) 1813 if self.p4branch: 1814 client_spec += "%s/" % (self.p4branch) 1815 client_spec += "... //%s/%s/...\n" % (self.p4client, self.srcdir) 1816 if self.p4extra_views: 1817 for k, v in self.p4extra_views: 1818 client_spec += "\t%s/... //%s/%s%s/...\n" % (k, self.p4client, 1819 self.srcdir, v) 1820 if self.p4port: 1821 command.extend(['-p', self.p4port]) 1822 if self.p4user: 1823 command.extend(['-u', self.p4user]) 1824 if self.p4passwd: 1825 command.extend(['-P', Obfuscated(self.p4passwd, "XXXXXXXX")]) 1826 command.extend(['client', '-i']) 1827 log.msg(client_spec) 1828 1829 # from bdbaddog in github comments: 1830 # I'm pretty sure the issue is that perforce client specs can't be 1831 # non-ascii (unless you configure at initial config to be unicode). I 1832 # floated a question to perforce mailing list. From reading the 1833 # internationalization notes.. 1834 # http://www.perforce.com/perforce/doc.092/user/i18nnotes.txt 1835 # I'm 90% sure that's the case. 1836 # (http://github.com/bdbaddog/buildbot/commit/8420149b2b804efcf5f81a13e18aa62da0424d21) 1837 1838 # Clean client spec to plain ascii 1839 client_spec=client_spec.encode('ascii','ignore') 1840 1841 c = ShellCommand(self.builder, command, self.builder.basedir, 1842 environ=env, sendRC=False, timeout=self.timeout, 1843 maxTime=self.maxTime, initialStdin=client_spec, 1844 usePTY=False) 1845 self.command = c 1846 d = c.start() 1847 d.addCallback(self._abandonOnFailure) 1848 d.addCallback(lambda _: self._doP4Sync(force=True)) 1849 return d
1850
1851 - def parseGotRevision(self):
1852 if self.revision: 1853 return str(self.revision) 1854 else: 1855 return P4Base.parseGotRevision(self)
1856 1857 registerSlaveCommand("p4", P4, command_version) 1858 1859
1860 -class P4Sync(P4Base):
1861 """A partial P4 source-updater. Requires manual setup of a per-slave P4 1862 environment. The only thing which comes from the master is P4PORT. 1863 'mode' is required to be 'copy'. 1864 1865 ['p4port'] (required): host:port for server to access 1866 ['p4user'] (optional): user to use for access 1867 ['p4passwd'] (optional): passwd to try for the user 1868 ['p4client'] (optional): client spec to use 1869 """ 1870 1871 header = "p4 sync" 1872
1873 - def setup(self, args):
1874 P4Base.setup(self, args) 1875 self.vcexe = getCommand("p4")
1876
1877 - def sourcedirIsUpdateable(self):
1878 return True
1879
1880 - def _doVC(self, force):
1881 d = os.path.join(self.builder.basedir, self.srcdir) 1882 command = [self.vcexe] 1883 if self.p4port: 1884 command.extend(['-p', self.p4port]) 1885 if self.p4user: 1886 command.extend(['-u', self.p4user]) 1887 if self.p4passwd: 1888 command.extend(['-P', Obfuscated(self.p4passwd, "XXXXXXXX")]) 1889 if self.p4client: 1890 command.extend(['-c', self.p4client]) 1891 command.extend(['sync']) 1892 if force: 1893 command.extend(['-f']) 1894 if self.revision: 1895 command.extend(['@' + self.revision]) 1896 env = {} 1897 c = ShellCommand(self.builder, command, d, environ=env, 1898 sendRC=False, timeout=self.timeout, 1899 maxTime=self.maxTime, usePTY=False) 1900 self.command = c 1901 return c.start()
1902
1903 - def doVCUpdate(self):
1904 return self._doVC(force=False)
1905
1906 - def doVCFull(self):
1907 return self._doVC(force=True)
1908
1909 - def parseGotRevision(self):
1910 if self.revision: 1911 return str(self.revision) 1912 else: 1913 return P4Base.parseGotRevision(self)
1914 1915 registerSlaveCommand("p4sync", P4Sync, command_version) 1916