1 import os, re, sys, shutil, time
2
3 from xml.dom.minidom import parseString
4
5 from twisted.python import log, failure, runtime
6 from twisted.internet import defer
7
8 from buildslave.commands.base import Command, ShellCommand, AbandonChain, command_version, Obfuscated
9 from buildslave.commands.registry import registerSlaveCommand
10 from buildslave.commands.utils import getCommand, rmdirRecursive
11 from buildslave.util import remove_userpassword
12
14 """Abstract base class for Version Control System operations (checkout
15 and update). This class extracts the following arguments from the
16 dictionary received from the master:
17
18 - ['workdir']: (required) the subdirectory where the buildable sources
19 should be placed
20
21 - ['mode']: one of update/copy/clobber/export, defaults to 'update'
22
23 - ['revision']: If not None, this is an int or string which indicates
24 which sources (along a time-like axis) should be used.
25 It is the thing you provide as the CVS -r or -D
26 argument.
27
28 - ['patch']: If not None, this is a tuple of (striplevel, patch)
29 which contains a patch that should be applied after the
30 checkout has occurred. Once applied, the tree is no
31 longer eligible for use with mode='update', and it only
32 makes sense to use this in conjunction with a
33 ['revision'] argument. striplevel is an int, and patch
34 is a string in standard unified diff format. The patch
35 will be applied with 'patch -p%d <PATCH', with
36 STRIPLEVEL substituted as %d. The command will fail if
37 the patch process fails (rejected hunks).
38
39 - ['timeout']: seconds of silence tolerated before we kill off the
40 command
41
42 - ['maxTime']: seconds before we kill off the command
43
44 - ['retry']: If not None, this is a tuple of (delay, repeats)
45 which means that any failed VC updates should be
46 reattempted, up to REPEATS times, after a delay of
47 DELAY seconds. This is intended to deal with slaves
48 that experience transient network failures.
49 """
50
51 sourcedata = ""
52
54
55
56
57 self.env = os.environ.copy()
58 self.env['LC_MESSAGES'] = "C"
59
60 self.workdir = args['workdir']
61 self.mode = args.get('mode', "update")
62 self.revision = args.get('revision')
63 self.patch = args.get('patch')
64 self.timeout = args.get('timeout', 120)
65 self.maxTime = args.get('maxTime', None)
66 self.retry = args.get('retry')
67
68
69
99
104
109
110 - def doVC(self, res):
123
125 try:
126 olddata = self.readSourcedata()
127 if olddata != self.sourcedata:
128 return False
129 except IOError:
130 return False
131 return True
132
137
139 d = defer.maybeDeferred(self.parseGotRevision)
140 d.addCallback(lambda got_revision:
141 self.sendStatus({'got_revision': got_revision}))
142 return d
143
145 """Override this in a subclass. It should return a string that
146 represents which revision was actually checked out, or a Deferred
147 that will fire with such a string. If, in a future build, you were to
148 pass this 'got_revision' string in as the 'revision' component of a
149 SourceStamp, you should wind up with the same source code as this
150 checkout just obtained.
151
152 It is probably most useful to scan self.command.stdout for a string
153 of some sort. Be sure to set keepStdout=True on the VC command that
154 you run, so that you'll have something available to look at.
155
156 If this information is unavailable, just return None."""
157
158 return None
159
161 return open(self.sourcedatafile, "r").read()
162
166
168 """Returns True if the tree can be updated."""
169 raise NotImplementedError("this must be implemented in a subclass")
170
172 """Returns a deferred with the steps to update a checkout."""
173 raise NotImplementedError("this must be implemented in a subclass")
174
176 """Returns a deferred with the steps to do a fresh checkout."""
177 raise NotImplementedError("this must be implemented in a subclass")
178
190
192 msg = "now retrying VC operation"
193 self.sendStatus({'header': msg + "\n"})
194 log.msg(msg)
195 d = self.doVCFull()
196 d.addBoth(self.maybeDoVCRetry)
197 d.addCallback(self._abandonOnFailure)
198 return d
199
201 """We get here somewhere after a VC chain has finished. res could
202 be::
203
204 - 0: the operation was successful
205 - nonzero: the operation failed. retry if possible
206 - AbandonChain: the operation failed, someone else noticed. retry.
207 - Failure: some other exception, re-raise
208 """
209
210 if isinstance(res, failure.Failure):
211 if self.interrupted:
212 return res
213 res.trap(AbandonChain)
214 else:
215 if type(res) is int and res == 0:
216 return res
217 if self.interrupted:
218 raise AbandonChain(1)
219
220 if self.retry:
221 delay, repeats = self.retry
222 if repeats >= 0:
223 self.retry = (delay, repeats-1)
224 msg = ("update failed, trying %d more times after %d seconds"
225 % (repeats, delay))
226 self.sendStatus({'header': msg + "\n"})
227 log.msg(msg)
228 d = defer.Deferred()
229
230
231 self.doClobber(d, self.workdir)
232 if self.srcdir:
233 self.doClobber(d, self.srcdir)
234 d.addCallback(lambda res: self.doVCFull())
235 d.addBoth(self.maybeDoVCRetry)
236 self._reactor.callLater(delay, d.callback, None)
237 return d
238 return res
239
240 - def doClobber(self, dummy, dirname, chmodDone=False):
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259 d = os.path.join(self.builder.basedir, dirname)
260 if runtime.platformType != "posix":
261
262
263 rmdirRecursive(d)
264 return defer.succeed(0)
265 command = ["rm", "-rf", d]
266 c = ShellCommand(self.builder, command, self.builder.basedir,
267 sendRC=0, timeout=self.timeout, maxTime=self.maxTime,
268 usePTY=False)
269
270 self.command = c
271
272
273
274 d = c.start()
275
276
277
278 if chmodDone:
279 d.addCallback(self._abandonOnFailure)
280 else:
281 d.addCallback(lambda rc: self.doClobberTryChmodIfFail(rc, dirname))
282 return d
283
285 assert isinstance(rc, int)
286 if rc == 0:
287 return defer.succeed(0)
288
289
290 command = ["chmod", "-Rf", "u+rwx", os.path.join(self.builder.basedir, dirname)]
291 if sys.platform.startswith('freebsd'):
292
293
294
295 command = ["find", os.path.join(self.builder.basedir, dirname),
296 '-exec', 'chmod', 'u+rwx', '{}', ';' ]
297 c = ShellCommand(self.builder, command, self.builder.basedir,
298 sendRC=0, timeout=self.timeout, maxTime=self.maxTime,
299 usePTY=False)
300
301 self.command = c
302 d = c.start()
303 d.addCallback(self._abandonOnFailure)
304 d.addCallback(lambda dummy: self.doClobber(dummy, dirname, True))
305 return d
306
308
309 fromdir = os.path.join(self.builder.basedir, self.srcdir)
310 todir = os.path.join(self.builder.basedir, self.workdir)
311 if runtime.platformType != "posix":
312 self.sendStatus({'header': "Since we're on a non-POSIX platform, "
313 "we're not going to try to execute cp in a subprocess, but instead "
314 "use shutil.copytree(), which will block until it is complete. "
315 "fromdir: %s, todir: %s\n" % (fromdir, todir)})
316 shutil.copytree(fromdir, todir)
317 return defer.succeed(0)
318
319 if not os.path.exists(os.path.dirname(todir)):
320 os.makedirs(os.path.dirname(todir))
321 if os.path.exists(todir):
322
323 log.msg("cp target '%s' already exists -- cp will not do what you think!" % todir)
324
325 command = ['cp', '-R', '-P', '-p', fromdir, todir]
326 c = ShellCommand(self.builder, command, self.builder.basedir,
327 sendRC=False, timeout=self.timeout, maxTime=self.maxTime,
328 usePTY=False)
329 self.command = c
330 d = c.start()
331 d.addCallback(self._abandonOnFailure)
332 return d
333
335 patchlevel = self.patch[0]
336 diff = self.patch[1]
337 root = None
338 if len(self.patch) >= 3:
339 root = self.patch[2]
340 command = [
341 getCommand("patch"),
342 '-p%d' % patchlevel,
343 '--remove-empty-files',
344 '--force',
345 '--forward',
346 ]
347 dir = os.path.join(self.builder.basedir, self.workdir)
348
349
350 marker = open(os.path.join(dir, ".buildbot-patched"), "w")
351 marker.write("patched\n")
352 marker.close()
353
354
355
356 if (root and
357 os.path.abspath(os.path.join(dir, root)
358 ).startswith(os.path.abspath(dir))):
359 dir = os.path.join(dir, root)
360
361
362 c = ShellCommand(self.builder, command, dir,
363 sendRC=False, timeout=self.timeout,
364 maxTime=self.maxTime, initialStdin=diff, usePTY=False)
365 self.command = c
366 d = c.start()
367 d.addCallback(self._abandonOnFailure)
368 return d
369
370
371
372 -class BK(SourceBase):
373 """BitKeeper-specific VC operation. In addition to the arguments
374 handled by SourceBase, this command reads the following keys:
375
376 ['bkurl'] (required): the BK repository string
377 """
378
379 header = "bk operation"
380
382 SourceBase.setup(self, args)
383 self.vcexe = getCommand("bk")
384 self.bkurl = args['bkurl']
385 self.sourcedata = '"%s\n"' % self.bkurl
386
387 self.bk_args = []
388 if args.get('extra_args', None) is not None:
389 self.bk_args.extend(args['extra_args'])
390
397
410
426
428 """
429 Get the (shell) command used to determine BK revision number
430 of checked-out code
431
432 return: list of strings, passable as the command argument to ShellCommand
433 """
434 return [self.vcexe, "changes", "-r+", "-d:REV:"]
435
437 c = ShellCommand(self.builder,
438 self.getBKVersionCommand(),
439 os.path.join(self.builder.basedir, self.srcdir),
440 environ=self.env,
441 sendStdout=False, sendStderr=False, sendRC=False,
442 keepStdout=True, usePTY=False)
443 d = c.start()
444 def _parse(res):
445 r_raw = c.stdout.strip()
446 got_version = None
447 try:
448 r = r_raw
449 except:
450 msg = ("BK.parseGotRevision unable to parse output: (%s)" % r_raw)
451 log.msg(msg)
452 self.sendStatus({'header': msg + "\n"})
453 raise ValueError(msg)
454 return r
455 d.addCallback(_parse)
456 return d
457
458 registerSlaveCommand("bk", BK, command_version)
459
460
461
462
463 -class CVS(SourceBase):
464 """CVS-specific VC operation. In addition to the arguments handled by
465 SourceBase, this command reads the following keys:
466
467 ['cvsroot'] (required): the CVSROOT repository string
468 ['cvsmodule'] (required): the module to be retrieved
469 ['branch']: a '-r' tag or branch name to use for the checkout/update
470 ['login']: a string for use as a password to 'cvs login'
471 ['global_options']: a list of strings to use before the CVS verb
472 ['checkout_options']: a list of strings to use after checkout,
473 but before revision and branch specifiers
474 ['checkout_options']: a list of strings to use after export,
475 but before revision and branch specifiers
476 ['extra_options']: a list of strings to use after export and checkout,
477 but before revision and branch specifiers
478 """
479
480 header = "cvs operation"
481
483 SourceBase.setup(self, args)
484 self.vcexe = getCommand("cvs")
485 self.cvsroot = args['cvsroot']
486 self.cvsmodule = args['cvsmodule']
487 self.global_options = args.get('global_options', [])
488 self.checkout_options = args.get('checkout_options', [])
489 self.export_options = args.get('export_options', [])
490 self.extra_options = args.get('extra_options', [])
491 self.branch = args.get('branch')
492 self.login = args.get('login')
493 self.sourcedata = "%s\n%s\n%s\n" % (self.cvsroot, self.cvsmodule,
494 self.branch)
495
500
518
522
524 d = os.path.join(self.builder.basedir, self.srcdir)
525 command = [self.vcexe, '-z3'] + self.global_options + ['update', '-dP']
526 if self.branch:
527 command += ['-r', self.branch]
528 if self.revision:
529 command += ['-D', self.revision]
530 c = ShellCommand(self.builder, command, d,
531 sendRC=False, timeout=self.timeout,
532 maxTime=self.maxTime, usePTY=False)
533 self.command = c
534 return c.start()
535
537 d = self.builder.basedir
538 if self.mode == "export":
539 verb = "export"
540 else:
541 verb = "checkout"
542 command = ([self.vcexe, '-d', self.cvsroot, '-z3'] +
543 self.global_options +
544 [verb, '-d', self.srcdir])
545
546 if verb == "checkout":
547 command += self.checkout_options
548 else:
549 command += self.export_options
550 command += self.extra_options
551
552 if self.branch:
553 command += ['-r', self.branch]
554 if self.revision:
555 command += ['-D', self.revision]
556 command += [self.cvsmodule]
557
558 c = ShellCommand(self.builder, command, d,
559 sendRC=False, timeout=self.timeout,
560 maxTime=self.maxTime, usePTY=False)
561 self.command = c
562 return c.start()
563
565
566
567
568
569 return time.strftime("%Y-%m-%d %H:%M:%S +0000", time.gmtime())
570
571 registerSlaveCommand("cvs", CVS, command_version)
572
573 -class SVN(SourceBase):
574 """Subversion-specific VC operation. In addition to the arguments
575 handled by SourceBase, this command reads the following keys:
576
577 ['svnurl'] (required): the SVN repository string
578 ['username']: Username passed to the svn command
579 ['password']: Password passed to the svn command
580 ['keep_on_purge']: Files and directories to keep between updates
581 ['ignore_ignores']: Ignore ignores when purging changes
582 ['always_purge']: Always purge local changes after each build
583 ['depth']: Pass depth argument to subversion 1.5+
584 """
585
586 header = "svn operation"
587
589 SourceBase.setup(self, args)
590 self.vcexe = getCommand("svn")
591 self.svnurl = args['svnurl']
592 self.sourcedata = "%s\n" % self.svnurl
593 self.keep_on_purge = args.get('keep_on_purge', [])
594 self.keep_on_purge.append(".buildbot-sourcedata")
595 self.ignore_ignores = args.get('ignore_ignores', True)
596 self.always_purge = args.get('always_purge', False)
597
598 self.svn_args = []
599 if args.has_key('username'):
600 self.svn_args.extend(["--username", args['username']])
601 if args.has_key('password'):
602 self.svn_args.extend(["--password", Obfuscated(args['password'], "XXXX")])
603 if args.get('extra_args', None) is not None:
604 self.svn_args.extend(args['extra_args'])
605
606 if args.has_key('depth'):
607 self.svn_args.extend(["--depth",args['depth']])
608
609 - def _dovccmd(self, command, args, rootdir=None, cb=None, **kwargs):
610 if rootdir is None:
611 rootdir = os.path.join(self.builder.basedir, self.srcdir)
612 fullCmd = [self.vcexe, command, '--non-interactive', '--no-auth-cache']
613 fullCmd.extend(self.svn_args)
614 fullCmd.extend(args)
615 c = ShellCommand(self.builder, fullCmd, rootdir,
616 environ=self.env, sendRC=False, timeout=self.timeout,
617 maxTime=self.maxTime, usePTY=False, **kwargs)
618 self.command = c
619 d = c.start()
620 if cb:
621 d.addCallback(self._abandonOnFailure)
622 d.addCallback(cb)
623 return d
624
628
630 if self.sourcedirIsPatched() or self.always_purge:
631 return self._purgeAndUpdate()
632 revision = self.args['revision'] or 'HEAD'
633
634 return self._dovccmd('update', ['--revision', str(revision)],
635 keepStdout=True)
636
638 revision = self.args['revision'] or 'HEAD'
639 args = ['--revision', str(revision), self.svnurl, self.srcdir]
640 if self.mode == "export":
641 command = 'export'
642 else:
643
644 command = 'checkout'
645 return self._dovccmd(command, args, rootdir=self.builder.basedir,
646 keepStdout=True)
647
649 """svn revert has several corner cases that make it unpractical.
650
651 Use the Force instead and delete everything that shows up in status."""
652 args = ['--xml']
653 if self.ignore_ignores:
654 args.append('--no-ignore')
655 return self._dovccmd('status', args, keepStdout=True, sendStdout=False,
656 cb=self._purgeAndUpdate2)
657
659 """Delete everything that shown up on status."""
660 result_xml = parseString(self.command.stdout)
661 for entry in result_xml.getElementsByTagName('entry'):
662 filename = entry.getAttribute('path')
663 if filename in self.keep_on_purge:
664 continue
665 filepath = os.path.join(self.builder.basedir, self.workdir,
666 filename)
667 self.sendStatus({'stdout': "%s\n" % filepath})
668 if os.path.isfile(filepath):
669 os.chmod(filepath, 0700)
670 os.remove(filepath)
671 else:
672 rmdirRecursive(filepath)
673
674 revision = self.args['revision'] or 'HEAD'
675 return self._dovccmd('update', ['--revision', str(revision)],
676 keepStdout=True)
677
679 """
680 Get the (shell) command used to determine SVN revision number
681 of checked-out code
682
683 return: list of strings, passable as the command argument to ShellCommand
684 """
685
686
687
688 svnversion_command = getCommand("svnversion")
689
690
691 return [svnversion_command, "."]
692
694 c = ShellCommand(self.builder,
695 self.getSvnVersionCommand(),
696 os.path.join(self.builder.basedir, self.srcdir),
697 environ=self.env,
698 sendStdout=False, sendStderr=False, sendRC=False,
699 keepStdout=True, usePTY=False)
700 d = c.start()
701 def _parse(res):
702 r_raw = c.stdout.strip()
703
704 r = r_raw.rstrip('MS')
705 r = r.split(':')[-1]
706 got_version = None
707 try:
708 got_version = int(r)
709 except ValueError:
710 msg =("SVN.parseGotRevision unable to parse output "
711 "of svnversion: '%s'" % r_raw)
712 log.msg(msg)
713 self.sendStatus({'header': msg + "\n"})
714 return got_version
715 d.addCallback(_parse)
716 return d
717
718
719 registerSlaveCommand("svn", SVN, command_version)
720
722 """Darcs-specific VC operation. In addition to the arguments
723 handled by SourceBase, this command reads the following keys:
724
725 ['repourl'] (required): the Darcs repository string
726 """
727
728 header = "darcs operation"
729
736
743
745 assert not self.revision
746
747 d = os.path.join(self.builder.basedir, self.srcdir)
748 command = [self.vcexe, 'pull', '--all', '--verbose']
749 c = ShellCommand(self.builder, command, d,
750 sendRC=False, timeout=self.timeout,
751 maxTime=self.maxTime, usePTY=False)
752 self.command = c
753 return c.start()
754
756
757 d = self.builder.basedir
758 command = [self.vcexe, 'get', '--verbose', '--partial',
759 '--repo-name', self.srcdir]
760 if self.revision:
761
762 n = os.path.join(self.builder.basedir, ".darcs-context")
763 f = open(n, "wb")
764 f.write(self.revision)
765 f.close()
766
767 command.append('--context')
768 command.append(n)
769 command.append(self.repourl)
770
771 c = ShellCommand(self.builder, command, d,
772 sendRC=False, timeout=self.timeout,
773 maxTime=self.maxTime, usePTY=False)
774 self.command = c
775 d = c.start()
776 if self.revision:
777 d.addCallback(self.removeContextFile, n)
778 return d
779
780 - def removeContextFile(self, res, n):
781 os.unlink(n)
782 return res
783
785
786 command = [self.vcexe, "changes", "--context"]
787 c = ShellCommand(self.builder, command,
788 os.path.join(self.builder.basedir, self.srcdir),
789 environ=self.env,
790 sendStdout=False, sendStderr=False, sendRC=False,
791 keepStdout=True, usePTY=False)
792 d = c.start()
793 d.addCallback(lambda res: c.stdout)
794 return d
795
796 registerSlaveCommand("darcs", Darcs, command_version)
797
799 """Monotone-specific VC operation. In addition to the arguments handled
800 by SourceBase, this command reads the following keys:
801
802 ['server_addr'] (required): the address of the server to pull from
803 ['branch'] (required): the branch the revision is on
804 ['db_path'] (required): the local database path to use
805 ['revision'] (required): the revision to check out
806 ['monotone']: (required): path to monotone executable
807 """
808
809 header = "monotone operation"
810
812 SourceBase.setup(self, args)
813 self.server_addr = args["server_addr"]
814 self.branch = args["branch"]
815 self.db_path = args["db_path"]
816 self.revision = args["revision"]
817 self.monotone = args["monotone"]
818 self._made_fulls = False
819 self._pull_timeout = args["timeout"]
820
827
829 self._makefulls()
830 return (not self.sourcedirIsPatched() and
831 os.path.isfile(self.full_db_path) and
832 os.path.isdir(os.path.join(self.full_srcdir, "MT")))
833
835 return self._withFreshDb(self._doUpdate)
836
838
839 command = [self.monotone, "update",
840 "-r", self.revision,
841 "-b", self.branch]
842 c = ShellCommand(self.builder, command, self.full_srcdir,
843 sendRC=False, timeout=self.timeout,
844 maxTime=self.maxTime, usePTY=False)
845 self.command = c
846 return c.start()
847
849 return self._withFreshDb(self._doFull)
850
852 command = [self.monotone, "--db=" + self.full_db_path,
853 "checkout",
854 "-r", self.revision,
855 "-b", self.branch,
856 self.full_srcdir]
857 c = ShellCommand(self.builder, command, self.builder.basedir,
858 sendRC=False, timeout=self.timeout,
859 maxTime=self.maxTime, usePTY=False)
860 self.command = c
861 return c.start()
862
864 self._makefulls()
865
866 if os.path.isfile(self.full_db_path):
867
868
869 command = [self.monotone, "db", "migrate",
870 "--db=" + self.full_db_path]
871 else:
872
873
874 self._pull_timeout = max(self._pull_timeout, 3 * 60 * 60)
875 self.sendStatus({"header": "creating database %s\n"
876 % (self.full_db_path,)})
877 command = [self.monotone, "db", "init",
878 "--db=" + self.full_db_path]
879 c = ShellCommand(self.builder, command, self.builder.basedir,
880 sendRC=False, timeout=self.timeout,
881 maxTime=self.maxTime, usePTY=False)
882 self.command = c
883 d = c.start()
884 d.addCallback(self._abandonOnFailure)
885 d.addCallback(self._didDbInit)
886 d.addCallback(self._didPull, callback)
887 return d
888
890 command = [self.monotone, "--db=" + self.full_db_path,
891 "pull", "--ticker=dot", self.server_addr, self.branch]
892 c = ShellCommand(self.builder, command, self.builder.basedir,
893 sendRC=False, timeout=self._pull_timeout,
894 maxTime=self.maxTime, usePTY=False)
895 self.sendStatus({"header": "pulling %s from %s\n"
896 % (self.branch, self.server_addr)})
897 self.command = c
898 return c.start()
899
902
903 registerSlaveCommand("monotone", Monotone, command_version)
904
905
906 -class Git(SourceBase):
907 """Git specific VC operation. In addition to the arguments
908 handled by SourceBase, this command reads the following keys:
909
910 ['repourl'] (required): the upstream GIT repository string
911 ['branch'] (optional): which version (i.e. branch or tag) to
912 retrieve. Default: "master".
913 ['submodules'] (optional): whether to initialize and update
914 submodules. Default: False.
915 ['ignore_ignores']: ignore ignores when purging changes.
916 """
917
918 header = "git operation"
919
930
933
938
940 return os.path.isdir(os.path.join(self._fullSrcdir(), ".git"))
941
942 - def _dovccmd(self, command, cb=None, **kwargs):
952
953
954
955
956
957
959 try:
960 olddata = self.readSourcedata()
961 if not olddata.startswith(self.repourl+' '):
962 return False
963 except IOError:
964 return False
965 return True
966
968 command = ['submodule', 'foreach', 'git', 'clean', '-d', '-f']
969 if self.ignore_ignores:
970 command.append('-x')
971 return self._dovccmd(command)
972
974 return self._dovccmd(['submodule', 'update'], self._cleanSubmodules)
975
977 if self.submodules:
978 return self._dovccmd(['submodule', 'init'], self._updateSubmodules)
979 else:
980 return defer.succeed(0)
981
983
984
985 command = ['branch', '-M', self.branch]
986 return self._dovccmd(command, self._initSubmodules)
987
989 if self.revision:
990 head = self.revision
991 else:
992 head = 'FETCH_HEAD'
993
994
995
996 command = ['reset', '--hard', head]
997 return self._dovccmd(command, self._didHeadCheckout)
998
999
1000
1001
1002
1004 try:
1005
1006 diffbranch = self.sourcedata != self.readSourcedata()
1007 except IOError:
1008 diffbranch = False
1009 if diffbranch:
1010 command = ['clean', '-f', '-d']
1011 if self.ignore_ignores:
1012 command.append('-x')
1013 return self._dovccmd(command, self._didClean)
1014 return self._didClean(None)
1015
1017
1018
1019 command = ['fetch', '-t', self.repourl, '+%s' % self.branch]
1020 self.sendStatus({"header": "fetching branch %s from %s\n"
1021 % (self.branch, self.repourl)})
1022 return self._dovccmd(command, self._didFetch)
1023
1025
1026 if self.revision:
1027
1028 d = self._dovccmd(['reset', '--hard', self.revision],
1029 self._initSubmodules)
1030
1031
1032 d.addErrback(self._doFetch)
1033 return d
1034 else:
1035
1036 return self._doFetch(None)
1037
1040
1042
1043
1044 if not self.args.get('revision') and self.args.get('shallow'):
1045 cmd = [self.vcexe, 'clone', '--depth', '1', self.repourl,
1046 self._fullSrcdir()]
1047 c = ShellCommand(self.builder, cmd, self.builder.basedir,
1048 sendRC=False, timeout=self.timeout,
1049 maxTime=self.maxTime, usePTY=False)
1050 self.command = c
1051 cmdexec = c.start()
1052 cmdexec.addCallback(self._didInit)
1053 return cmdexec
1054 else:
1055 os.makedirs(self._fullSrcdir())
1056 return self._dovccmd(['init'], self._didInit)
1057
1059 command = ['rev-parse', 'HEAD']
1060 def _parse(res):
1061 hash = self.command.stdout.strip()
1062 if len(hash) != 40:
1063 return None
1064 return hash
1065 return self._dovccmd(command, _parse, keepStdout=True)
1066
1067 registerSlaveCommand("git", Git, command_version)
1068
1069 -class Arch(SourceBase):
1070 """Arch-specific (tla-specific) VC operation. In addition to the
1071 arguments handled by SourceBase, this command reads the following keys:
1072
1073 ['url'] (required): the repository string
1074 ['version'] (required): which version (i.e. branch) to retrieve
1075 ['revision'] (optional): the 'patch-NN' argument to check out
1076 ['archive']: the archive name to use. If None, use the archive's default
1077 ['build-config']: if present, give to 'tla build-config' after checkout
1078 """
1079
1080 header = "arch operation"
1081 buildconfig = None
1082
1093
1104
1116
1118
1119
1120
1121
1122
1123 command = [self.vcexe, 'register-archive', '--force', self.url]
1124 c = ShellCommand(self.builder, command, self.builder.basedir,
1125 sendRC=False, keepStdout=True, timeout=self.timeout,
1126 maxTime=self.maxTime, usePTY=False)
1127 self.command = c
1128 d = c.start()
1129 d.addCallback(self._abandonOnFailure)
1130 d.addCallback(self._didRegister, c)
1131 return d
1132
1134
1135
1136 r = re.search(r'Registering archive: (\S+)\s*$', c.stdout)
1137 if r:
1138 msg = "tla reports archive name is '%s'" % r.group(1)
1139 log.msg(msg)
1140 self.builder.sendUpdate({'header': msg+"\n"})
1141 if self.archive and r.group(1) != self.archive:
1142 msg = (" mismatch, we wanted an archive named '%s'"
1143 % self.archive)
1144 log.msg(msg)
1145 self.builder.sendUpdate({'header': msg+"\n"})
1146 raise AbandonChain(-1)
1147 self.archive = r.group(1)
1148 assert self.archive, "need archive name to continue"
1149 return self._doGet()
1150
1152 ver = self.version
1153 if self.revision:
1154 ver += "--%s" % self.revision
1155 command = [self.vcexe, 'get', '--archive', self.archive,
1156 '--no-pristine',
1157 ver, self.srcdir]
1158 c = ShellCommand(self.builder, command, self.builder.basedir,
1159 sendRC=False, timeout=self.timeout,
1160 maxTime=self.maxTime, usePTY=False)
1161 self.command = c
1162 d = c.start()
1163 d.addCallback(self._abandonOnFailure)
1164 if self.buildconfig:
1165 d.addCallback(self._didGet)
1166 return d
1167
1178
1180
1181
1182
1183 command = [self.vcexe, "logs", "--full", "--reverse"]
1184 c = ShellCommand(self.builder, command,
1185 os.path.join(self.builder.basedir, self.srcdir),
1186 environ=self.env,
1187 sendStdout=False, sendStderr=False, sendRC=False,
1188 keepStdout=True, usePTY=False)
1189 d = c.start()
1190 def _parse(res):
1191 tid = c.stdout.split("\n")[0].strip()
1192 slash = tid.index("/")
1193 dd = tid.rindex("--")
1194
1195 baserev = tid[dd+2:]
1196 return baserev
1197 d.addCallback(_parse)
1198 return d
1199
1200 registerSlaveCommand("arch", Arch, command_version)
1201
1203 """Bazaar (/usr/bin/baz) is an alternative client for Arch repositories.
1204 It is mostly option-compatible, but archive registration is different
1205 enough to warrant a separate Command.
1206
1207 ['archive'] (required): the name of the archive being used
1208 """
1209
1220
1221
1222
1223
1225
1226
1227 ver = self.archive + "/" + self.version
1228 if self.revision:
1229 ver += "--%s" % self.revision
1230 command = [self.vcexe, 'get', '--no-pristine',
1231 ver, self.srcdir]
1232 c = ShellCommand(self.builder, command, self.builder.basedir,
1233 sendRC=False, timeout=self.timeout,
1234 maxTime=self.maxTime, usePTY=False)
1235 self.command = c
1236 d = c.start()
1237 d.addCallback(self._abandonOnFailure)
1238 if self.buildconfig:
1239 d.addCallback(self._didGet)
1240 return d
1241
1243
1244 command = [self.vcexe, "tree-id"]
1245 c = ShellCommand(self.builder, command,
1246 os.path.join(self.builder.basedir, self.srcdir),
1247 environ=self.env,
1248 sendStdout=False, sendStderr=False, sendRC=False,
1249 keepStdout=True, usePTY=False)
1250 d = c.start()
1251 def _parse(res):
1252 tid = c.stdout.strip()
1253 slash = tid.index("/")
1254 dd = tid.rindex("--")
1255
1256 baserev = tid[dd+2:]
1257 return baserev
1258 d.addCallback(_parse)
1259 return d
1260
1261 registerSlaveCommand("bazaar", Bazaar, command_version)
1262
1263
1264 -class Bzr(SourceBase):
1265 """bzr-specific VC operation. In addition to the arguments
1266 handled by SourceBase, this command reads the following keys:
1267
1268 ['repourl'] (required): the Bzr repository string
1269 """
1270
1271 header = "bzr operation"
1272
1280
1287
1292
1293 if self.forceSharedRepo:
1294 d = self.doForceSharedRepo();
1295 d.addCallback(cont)
1296 return d
1297 else:
1298 return cont(None)
1299
1310
1340
1342 tmpdir = os.path.join(self.builder.basedir, "export-temp")
1343 srcdir = os.path.join(self.builder.basedir, self.srcdir)
1344 command = [self.vcexe, 'checkout', '--lightweight']
1345 if self.revision:
1346 command.append('--revision')
1347 command.append(str(self.revision))
1348 command.append(self.repourl)
1349 command.append(tmpdir)
1350 c = ShellCommand(self.builder, command, self.builder.basedir,
1351 sendRC=False, timeout=self.timeout,
1352 maxTime=self.maxTime, usePTY=False)
1353 self.command = c
1354 d = c.start()
1355 def _export(res):
1356 command = [self.vcexe, 'export', srcdir]
1357 c = ShellCommand(self.builder, command, tmpdir,
1358 sendRC=False, timeout=self.timeout,
1359 maxTime=self.maxTime, usePTY=False)
1360 self.command = c
1361 return c.start()
1362 d.addCallback(_export)
1363 return d
1364
1366
1367
1368
1369 c = ShellCommand(self.builder, [self.vcexe, 'info', '.'],
1370 self.builder.basedir,
1371 sendStderr=False, sendRC=False, usePTY=False)
1372 d = c.start()
1373 def afterCheckSharedRepo(res):
1374 if type(res) is int and res != 0:
1375 log.msg("No shared repo found, creating it")
1376
1377 c = ShellCommand(self.builder, [self.vcexe, 'init-repo', '.'],
1378 self.builder.basedir,
1379 sendRC=False, usePTY=False)
1380 self.command = c
1381 return c.start()
1382 else:
1383 return defer.succeed(res)
1384 d.addCallback(afterCheckSharedRepo)
1385 return d
1386
1388
1389
1390
1391 for line in out.split("\n"):
1392 colon = line.find(":")
1393 if colon != -1:
1394 key, value = line[:colon], line[colon+2:]
1395 if key == "revno":
1396 return int(value)
1397 raise ValueError("unable to find revno: in bzr output: '%s'" % out)
1398
1400 command = [self.vcexe, "version-info"]
1401 c = ShellCommand(self.builder, command,
1402 os.path.join(self.builder.basedir, self.srcdir),
1403 environ=self.env,
1404 sendStdout=False, sendStderr=False, sendRC=False,
1405 keepStdout=True, usePTY=False)
1406 d = c.start()
1407 def _parse(res):
1408 try:
1409 return self.get_revision_number(c.stdout)
1410 except ValueError:
1411 msg =("Bzr.parseGotRevision unable to parse output "
1412 "of bzr version-info: '%s'" % c.stdout.strip())
1413 log.msg(msg)
1414 self.sendStatus({'header': msg + "\n"})
1415 return None
1416 d.addCallback(_parse)
1417 return d
1418
1419 registerSlaveCommand("bzr", Bzr, command_version)
1420
1422 """Mercurial specific VC operation. In addition to the arguments
1423 handled by SourceBase, this command reads the following keys:
1424
1425 ['repourl'] (required): the Mercurial repository string
1426 ['clobberOnBranchChange']: Document me. See ticket #462.
1427 """
1428
1429 header = "mercurial operation"
1430
1432 SourceBase.setup(self, args)
1433 self.vcexe = getCommand("hg")
1434 self.repourl = args['repourl']
1435 self.clobberOnBranchChange = args.get('clobberOnBranchChange', True)
1436 self.sourcedata = "%s\n" % self.repourl
1437 self.branchType = args.get('branchType', 'dirname')
1438 self.stdout = ""
1439 self.stderr = ""
1440 self.clobbercount = 0
1441
1445
1447 d = os.path.join(self.builder.basedir, self.srcdir)
1448 command = [self.vcexe, 'pull', '--verbose', self.repourl]
1449 c = ShellCommand(self.builder, command, d,
1450 sendRC=False, timeout=self.timeout,
1451 maxTime=self.maxTime, keepStdout=True, usePTY=False)
1452 self.command = c
1453 d = c.start()
1454 d.addCallback(self._handleEmptyUpdate)
1455 d.addCallback(self._update)
1456 return d
1457
1459 if type(res) is int and res == 1:
1460 if self.command.stdout.find("no changes found") != -1:
1461
1462
1463
1464
1465 return 0
1466 return res
1467
1469 d = os.path.join(self.builder.basedir, self.srcdir)
1470 command = [self.vcexe, 'clone', '--verbose', '--noupdate']
1471
1472
1473
1474 if self.args.get('revision') and self.mode == 'clobber' and self.branchType == 'dirname':
1475 command.extend(['--rev', self.args.get('revision')])
1476 command.extend([self.repourl, d])
1477
1478 c = ShellCommand(self.builder, command, self.builder.basedir,
1479 sendRC=False, timeout=self.timeout,
1480 maxTime=self.maxTime, usePTY=False)
1481 self.command = c
1482 cmd1 = c.start()
1483 cmd1.addCallback(self._update)
1484 return cmd1
1485
1487 self.clobbercount += 1
1488
1489 if self.clobbercount > 3:
1490 raise Exception, "Too many clobber attempts. Aborting step"
1491
1492 def _vcfull(res):
1493 return self.doVCFull()
1494
1495 c = self.doClobber(dummy, dirname)
1496 c.addCallback(_vcfull)
1497
1498 return c
1499
1500 - def _purge(self, dummy, dirname):
1501 d = os.path.join(self.builder.basedir, self.srcdir)
1502 purge = [self.vcexe, 'purge', '--all']
1503 purgeCmd = ShellCommand(self.builder, purge, d,
1504 sendStdout=False, sendStderr=False,
1505 keepStdout=True, keepStderr=True, usePTY=False)
1506
1507 def _clobber(res):
1508 if res != 0:
1509
1510 msg = "'hg purge' failed: %s\n%s. Clobbering." % (purgeCmd.stdout, purgeCmd.stderr)
1511 self.sendStatus({'header': msg + "\n"})
1512 log.msg(msg)
1513
1514 return self._clobber(dummy, dirname)
1515
1516
1517 return self._update2(res)
1518
1519 p = purgeCmd.start()
1520 p.addCallback(_clobber)
1521 return p
1522
1524 if res != 0:
1525 return res
1526
1527
1528 self.update_branch = self.args.get('branch', 'default')
1529
1530 d = os.path.join(self.builder.basedir, self.srcdir)
1531 parentscmd = [self.vcexe, 'identify', '--num', '--branch']
1532 cmd = ShellCommand(self.builder, parentscmd, d,
1533 sendStdout=False, sendStderr=False,
1534 keepStdout=True, keepStderr=True, usePTY=False)
1535
1536 self.clobber = None
1537
1538 def _parseIdentify(res):
1539 if res != 0:
1540 msg = "'hg identify' failed: %s\n%s" % (cmd.stdout, cmd.stderr)
1541 self.sendStatus({'header': msg + "\n"})
1542 log.msg(msg)
1543 return res
1544
1545 log.msg('Output: %s' % cmd.stdout)
1546
1547 match = re.search(r'^(.+) (.+)$', cmd.stdout)
1548 assert match
1549
1550 rev = match.group(1)
1551 current_branch = match.group(2)
1552
1553 if rev == '-1':
1554 msg = "Fresh hg repo, don't worry about in-repo branch name"
1555 log.msg(msg)
1556
1557 elif self.sourcedirIsPatched():
1558 self.clobber = self._purge
1559
1560 elif self.update_branch != current_branch:
1561 msg = "Working dir is on in-repo branch '%s' and build needs '%s'." % (current_branch, self.update_branch)
1562 if self.clobberOnBranchChange:
1563 msg += ' Cloberring.'
1564 else:
1565 msg += ' Updating.'
1566
1567 self.sendStatus({'header': msg + "\n"})
1568 log.msg(msg)
1569
1570
1571 if self.clobberOnBranchChange:
1572 self.clobber = self._purge
1573
1574 else:
1575 msg = "Working dir on same in-repo branch as build (%s)." % (current_branch)
1576 log.msg(msg)
1577
1578 return 0
1579
1580 def _checkRepoURL(res):
1581 parentscmd = [self.vcexe, 'paths', 'default']
1582 cmd2 = ShellCommand(self.builder, parentscmd, d,
1583 sendStdout=False, sendStderr=False,
1584 keepStdout=True, keepStderr=True, usePTY=False)
1585
1586 def _parseRepoURL(res):
1587 if res == 1:
1588 if "not found!" == cmd2.stderr.strip():
1589 msg = "hg default path not set. Not checking repo url for clobber test"
1590 log.msg(msg)
1591 return 0
1592 else:
1593 msg = "'hg paths default' failed: %s\n%s" % (cmd2.stdout, cmd2.stderr)
1594 log.msg(msg)
1595 return 1
1596
1597 oldurl = cmd2.stdout.strip()
1598
1599 log.msg("Repo cloned from: '%s'" % oldurl)
1600
1601 if runtime.platformType == 'win32':
1602 oldurl = oldurl.lower().replace('\\', '/')
1603 repourl = self.repourl.lower().replace('\\', '/')
1604 else:
1605 repourl = self.repourl
1606
1607 if repourl.startswith('file://'):
1608 repourl = repourl.split('file://')[1]
1609 if oldurl.startswith('file://'):
1610 oldurl = oldurl.split('file://')[1]
1611
1612 oldurl = remove_userpassword(oldurl)
1613 repourl = remove_userpassword(repourl)
1614
1615 if oldurl.rstrip('/') != repourl.rstrip('/'):
1616 self.clobber = self._clobber
1617 msg = "RepoURL changed from '%s' in wc to '%s' in update. Clobbering" % (oldurl, repourl)
1618 log.msg(msg)
1619
1620 return 0
1621
1622 c = cmd2.start()
1623 c.addCallback(_parseRepoURL)
1624 return c
1625
1626 def _maybeClobber(res):
1627 if self.clobber:
1628 msg = "Clobber flag set. Doing clobbering"
1629 log.msg(msg)
1630
1631 def _vcfull(res):
1632 return self.doVCFull()
1633
1634 return self.clobber(None, self.srcdir)
1635
1636 return 0
1637
1638 c = cmd.start()
1639 c.addCallback(_parseIdentify)
1640 c.addCallback(_checkRepoURL)
1641 c.addCallback(_maybeClobber)
1642 c.addCallback(self._update2)
1643 return c
1644
1646 d = os.path.join(self.builder.basedir, self.srcdir)
1647
1648 updatecmd=[self.vcexe, 'update', '--clean', '--repository', d]
1649 if self.args.get('revision'):
1650 updatecmd.extend(['--rev', self.args['revision']])
1651 else:
1652 updatecmd.extend(['--rev', self.args.get('branch', 'default')])
1653 self.command = ShellCommand(self.builder, updatecmd,
1654 self.builder.basedir, sendRC=False,
1655 timeout=self.timeout, maxTime=self.maxTime, usePTY=False)
1656 return self.command.start()
1657
1659
1660 command = [self.vcexe, "identify", "--id", "--debug"]
1661 c = ShellCommand(self.builder, command,
1662 os.path.join(self.builder.basedir, self.srcdir),
1663 environ=self.env,
1664 sendStdout=False, sendStderr=False, sendRC=False,
1665 keepStdout=True, usePTY=False)
1666 d = c.start()
1667 def _parse(res):
1668 m = re.search(r'^(\w+)', c.stdout)
1669 return m.group(1)
1670 d.addCallback(_parse)
1671 return d
1672
1673 registerSlaveCommand("hg", Mercurial, command_version)
1674
1675
1677 """Base class for P4 source-updaters
1678
1679 ['p4port'] (required): host:port for server to access
1680 ['p4user'] (optional): user to use for access
1681 ['p4passwd'] (optional): passwd to try for the user
1682 ['p4client'] (optional): client spec to use
1683 """
1685 SourceBase.setup(self, args)
1686 self.p4port = args['p4port']
1687 self.p4client = args['p4client']
1688 self.p4user = args['p4user']
1689 self.p4passwd = args['p4passwd']
1690
1692
1693
1694 command = ['p4']
1695 if self.p4port:
1696 command.extend(['-p', self.p4port])
1697 if self.p4user:
1698 command.extend(['-u', self.p4user])
1699 if self.p4passwd:
1700 command.extend(['-P', Obfuscated(self.p4passwd, "XXXXXXXX")])
1701 if self.p4client:
1702 command.extend(['-c', self.p4client])
1703
1704 command.extend(['changes', '-s', 'submitted', '-m', '1', '#have'])
1705 c = ShellCommand(self.builder, command, self.builder.basedir,
1706 environ=self.env, timeout=self.timeout,
1707 maxTime=self.maxTime, sendStdout=True,
1708 sendStderr=False, sendRC=False, keepStdout=True,
1709 usePTY=False)
1710 self.command = c
1711 d = c.start()
1712
1713 def _parse(res):
1714
1715
1716
1717 m = re.match('Change\s+(\d+)\s+', c.stdout)
1718 if m:
1719 return m.group(1)
1720 return None
1721 d.addCallback(_parse)
1722 return d
1723
1724
1726 """A P4 source-updater.
1727
1728 ['p4port'] (required): host:port for server to access
1729 ['p4user'] (optional): user to use for access
1730 ['p4passwd'] (optional): passwd to try for the user
1731 ['p4client'] (optional): client spec to use
1732 ['p4extra_views'] (optional): additional client views to use
1733 """
1734
1735 header = "p4"
1736
1738 P4Base.setup(self, args)
1739 self.p4base = args['p4base']
1740 self.p4extra_views = args['p4extra_views']
1741 self.p4mode = args['mode']
1742 self.p4branch = args['branch']
1743
1744 self.sourcedata = str([
1745
1746 self.p4port,
1747
1748
1749 self.p4client,
1750
1751
1752 self.p4base,
1753 self.p4branch,
1754 self.p4extra_views,
1755
1756
1757 self.builder.basedir,
1758 self.mode,
1759 self.workdir
1760 ])
1761
1762
1770
1772 return self._doP4Sync(force=False)
1773
1775 command = ['p4']
1776
1777 if self.p4port:
1778 command.extend(['-p', self.p4port])
1779 if self.p4user:
1780 command.extend(['-u', self.p4user])
1781 if self.p4passwd:
1782 command.extend(['-P', Obfuscated(self.p4passwd, "XXXXXXXX")])
1783 if self.p4client:
1784 command.extend(['-c', self.p4client])
1785 command.extend(['sync'])
1786 if force:
1787 command.extend(['-f'])
1788 if self.revision:
1789 command.extend(['@' + str(self.revision)])
1790 env = {}
1791 c = ShellCommand(self.builder, command, self.builder.basedir,
1792 environ=env, sendRC=False, timeout=self.timeout,
1793 maxTime=self.maxTime, keepStdout=True, usePTY=False)
1794 self.command = c
1795 d = c.start()
1796 d.addCallback(self._abandonOnFailure)
1797 return d
1798
1799
1801 env = {}
1802 command = ['p4']
1803 client_spec = ''
1804 client_spec += "Client: %s\n\n" % self.p4client
1805 client_spec += "Owner: %s\n\n" % self.p4user
1806 client_spec += "Description:\n\tCreated by %s\n\n" % self.p4user
1807 client_spec += "Root:\t%s\n\n" % self.builder.basedir
1808 client_spec += "Options:\tallwrite rmdir\n\n"
1809 client_spec += "LineEnd:\tlocal\n\n"
1810
1811
1812 client_spec += "View:\n\t%s" % (self.p4base)
1813 if self.p4branch:
1814 client_spec += "%s/" % (self.p4branch)
1815 client_spec += "... //%s/%s/...\n" % (self.p4client, self.srcdir)
1816 if self.p4extra_views:
1817 for k, v in self.p4extra_views:
1818 client_spec += "\t%s/... //%s/%s%s/...\n" % (k, self.p4client,
1819 self.srcdir, v)
1820 if self.p4port:
1821 command.extend(['-p', self.p4port])
1822 if self.p4user:
1823 command.extend(['-u', self.p4user])
1824 if self.p4passwd:
1825 command.extend(['-P', Obfuscated(self.p4passwd, "XXXXXXXX")])
1826 command.extend(['client', '-i'])
1827 log.msg(client_spec)
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839 client_spec=client_spec.encode('ascii','ignore')
1840
1841 c = ShellCommand(self.builder, command, self.builder.basedir,
1842 environ=env, sendRC=False, timeout=self.timeout,
1843 maxTime=self.maxTime, initialStdin=client_spec,
1844 usePTY=False)
1845 self.command = c
1846 d = c.start()
1847 d.addCallback(self._abandonOnFailure)
1848 d.addCallback(lambda _: self._doP4Sync(force=True))
1849 return d
1850
1856
1857 registerSlaveCommand("p4", P4, command_version)
1858
1859
1861 """A partial P4 source-updater. Requires manual setup of a per-slave P4
1862 environment. The only thing which comes from the master is P4PORT.
1863 'mode' is required to be 'copy'.
1864
1865 ['p4port'] (required): host:port for server to access
1866 ['p4user'] (optional): user to use for access
1867 ['p4passwd'] (optional): passwd to try for the user
1868 ['p4client'] (optional): client spec to use
1869 """
1870
1871 header = "p4 sync"
1872
1876
1879
1880 - def _doVC(self, force):
1881 d = os.path.join(self.builder.basedir, self.srcdir)
1882 command = [self.vcexe]
1883 if self.p4port:
1884 command.extend(['-p', self.p4port])
1885 if self.p4user:
1886 command.extend(['-u', self.p4user])
1887 if self.p4passwd:
1888 command.extend(['-P', Obfuscated(self.p4passwd, "XXXXXXXX")])
1889 if self.p4client:
1890 command.extend(['-c', self.p4client])
1891 command.extend(['sync'])
1892 if force:
1893 command.extend(['-f'])
1894 if self.revision:
1895 command.extend(['@' + self.revision])
1896 env = {}
1897 c = ShellCommand(self.builder, command, d, environ=env,
1898 sendRC=False, timeout=self.timeout,
1899 maxTime=self.maxTime, usePTY=False)
1900 self.command = c
1901 return c.start()
1902
1904 return self._doVC(force=False)
1905
1907 return self._doVC(force=True)
1908
1914
1915 registerSlaveCommand("p4sync", P4Sync, command_version)
1916