1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 from warnings import warn
18 from email.Utils import formatdate
19 from twisted.python import log
20 from buildbot.process.buildstep import LoggingBuildStep, LoggedRemoteCommand
21 from buildbot.interfaces import BuildSlaveTooOldError
22 from buildbot.status.builder import SKIPPED
23
24
25 -class Source(LoggingBuildStep):
26 """This is a base class to generate a source tree in the buildslave.
27 Each version control system has a specialized subclass, and is expected
28 to override __init__ and implement computeSourceRevision() and
29 startVC(). The class as a whole builds up the self.args dictionary, then
30 starts a LoggedRemoteCommand with those arguments.
31 """
32
33
34 haltOnFailure = True
35 flunkOnFailure = True
36 notReally = False
37
38 branch = None
39
40 - def __init__(self, workdir=None, mode='update', alwaysUseLatest=False,
41 timeout=20*60, retry=None, **kwargs):
42 """
43 @type workdir: string
44 @param workdir: local directory (relative to the Builder's root)
45 where the tree should be placed
46
47 @type mode: string
48 @param mode: the kind of VC operation that is desired:
49 - 'update': specifies that the checkout/update should be
50 performed directly into the workdir. Each build is performed
51 in the same directory, allowing for incremental builds. This
52 minimizes disk space, bandwidth, and CPU time. However, it
53 may encounter problems if the build process does not handle
54 dependencies properly (if you must sometimes do a 'clean
55 build' to make sure everything gets compiled), or if source
56 files are deleted but generated files can influence test
57 behavior (e.g. python's .pyc files), or when source
58 directories are deleted but generated files prevent CVS from
59 removing them. When used with a patched checkout, from a
60 previous buildbot try for instance, it will try to "revert"
61 the changes first and will do a clobber if it is unable to
62 get a clean checkout. The behavior is SCM-dependent.
63
64 - 'copy': specifies that the source-controlled workspace
65 should be maintained in a separate directory (called the
66 'copydir'), using checkout or update as necessary. For each
67 build, a new workdir is created with a copy of the source
68 tree (rm -rf workdir; cp -R -P -p copydir workdir). This
69 doubles the disk space required, but keeps the bandwidth low
70 (update instead of a full checkout). A full 'clean' build
71 is performed each time. This avoids any generated-file
72 build problems, but is still occasionally vulnerable to
73 problems such as a CVS repository being manually rearranged
74 (causing CVS errors on update) which are not an issue with
75 a full checkout.
76
77 - 'clobber': specifies that the working directory should be
78 deleted each time, necessitating a full checkout for each
79 build. This insures a clean build off a complete checkout,
80 avoiding any of the problems described above, but is
81 bandwidth intensive, as the whole source tree must be
82 pulled down for each build.
83
84 - 'export': is like 'clobber', except that e.g. the 'cvs
85 export' command is used to create the working directory.
86 This command removes all VC metadata files (the
87 CVS/.svn/{arch} directories) from the tree, which is
88 sometimes useful for creating source tarballs (to avoid
89 including the metadata in the tar file). Not all VC systems
90 support export.
91
92 @type alwaysUseLatest: boolean
93 @param alwaysUseLatest: whether to always update to the most
94 recent available sources for this build.
95
96 Normally the Source step asks its Build for a list of all
97 Changes that are supposed to go into the build, then computes a
98 'source stamp' (revision number or timestamp) that will cause
99 exactly that set of changes to be present in the checked out
100 tree. This is turned into, e.g., 'cvs update -D timestamp', or
101 'svn update -r revnum'. If alwaysUseLatest=True, bypass this
102 computation and always update to the latest available sources
103 for each build.
104
105 The source stamp helps avoid a race condition in which someone
106 commits a change after the master has decided to start a build
107 but before the slave finishes checking out the sources. At best
108 this results in a build which contains more changes than the
109 buildmaster thinks it has (possibly resulting in the wrong
110 person taking the blame for any problems that result), at worst
111 is can result in an incoherent set of sources (splitting a
112 non-atomic commit) which may not build at all.
113
114 @type retry: tuple of ints (delay, repeats) (or None)
115 @param retry: if provided, VC update failures are re-attempted up
116 to REPEATS times, with DELAY seconds between each
117 attempt. Some users have slaves with poor connectivity
118 to their VC repository, and they say that up to 80% of
119 their build failures are due to transient network
120 failures that could be handled by simply retrying a
121 couple times.
122
123 """
124
125 LoggingBuildStep.__init__(self, **kwargs)
126 self.addFactoryArguments(workdir=workdir,
127 mode=mode,
128 alwaysUseLatest=alwaysUseLatest,
129 timeout=timeout,
130 retry=retry,
131 )
132
133 assert mode in ("update", "copy", "clobber", "export")
134 if retry:
135 delay, repeats = retry
136 assert isinstance(repeats, int)
137 assert repeats > 0
138 self.args = {'mode': mode,
139 'timeout': timeout,
140 'retry': retry,
141 'patch': None,
142 }
143
144 self.workdir = workdir
145
146 self.alwaysUseLatest = alwaysUseLatest
147
148
149 description = ["updating"]
150 descriptionDone = ["update"]
151 if mode == "clobber":
152 description = ["checkout"]
153
154 descriptionDone = ["checkout"]
155 elif mode == "export":
156 description = ["exporting"]
157 descriptionDone = ["export"]
158 self.description = description
159 self.descriptionDone = descriptionDone
160
163
166
171
173 """Each subclass must implement this method to do something more
174 precise than -rHEAD every time. For version control systems that use
175 repository-wide change numbers (SVN, P4), this can simply take the
176 maximum such number from all the changes involved in this build. For
177 systems that do not (CVS), it needs to create a timestamp based upon
178 the latest Change, the Build's treeStableTimer, and an optional
179 self.checkoutDelay value."""
180 return None
181
209
246
248 if cmd.updates.has_key("got_revision"):
249 got_revision = cmd.updates["got_revision"][-1]
250 if got_revision is not None:
251 self.setProperty("got_revision", str(got_revision), "Source")
252
253
254
256 """I perform BitKeeper checkout/update operations."""
257
258 name = 'bk'
259
260 - def __init__(self, bkurl=None, baseURL=None,
261 directory=None, extra_args=None, **kwargs):
262 """
263 @type bkurl: string
264 @param bkurl: the URL which points to the BitKeeper server.
265
266 @type baseURL: string
267 @param baseURL: if branches are enabled, this is the base URL to
268 which a branch name will be appended. It should
269 probably end in a slash. Use exactly one of
270 C{bkurl} and C{baseURL}.
271 """
272
273 self.bkurl = bkurl
274 self.baseURL = baseURL
275 self.extra_args = extra_args
276
277 Source.__init__(self, **kwargs)
278 self.addFactoryArguments(bkurl=bkurl,
279 baseURL=baseURL,
280 directory=directory,
281 extra_args=extra_args,
282 )
283
284 if bkurl and baseURL:
285 raise ValueError("you must use exactly one of bkurl and baseURL")
286
287
290
291
292 - def startVC(self, branch, revision, patch):
322
323
324
326 """I do CVS checkout/update operations.
327
328 Note: if you are doing anonymous/pserver CVS operations, you will need
329 to manually do a 'cvs login' on each buildslave before the slave has any
330 hope of success. XXX: fix then, take a cvs password as an argument and
331 figure out how to do a 'cvs login' on each build
332 """
333
334 name = "cvs"
335
336
337
338
339
340
341
342
343
344
345
346 - def __init__(self, cvsroot=None, cvsmodule="",
347 global_options=[], branch=None, checkoutDelay=None,
348 checkout_options=[], export_options=[], extra_options=[],
349 login=None,
350 **kwargs):
351
352 """
353 @type cvsroot: string
354 @param cvsroot: CVS Repository from which the source tree should
355 be obtained. '/home/warner/Repository' for local
356 or NFS-reachable repositories,
357 ':pserver:anon@foo.com:/cvs' for anonymous CVS,
358 'user@host.com:/cvs' for non-anonymous CVS or
359 CVS over ssh. Lots of possibilities, check the
360 CVS documentation for more.
361
362 @type cvsmodule: string
363 @param cvsmodule: subdirectory of CVS repository that should be
364 retrieved
365
366 @type login: string or None
367 @param login: if not None, a string which will be provided as a
368 password to the 'cvs login' command, used when a
369 :pserver: method is used to access the repository.
370 This login is only needed once, but must be run
371 each time (just before the CVS operation) because
372 there is no way for the buildslave to tell whether
373 it was previously performed or not.
374
375 @type branch: string
376 @param branch: the default branch name, will be used in a '-r'
377 argument to specify which branch of the source tree
378 should be used for this checkout. Defaults to None,
379 which means to use 'HEAD'.
380
381 @type checkoutDelay: int or None
382 @param checkoutDelay: if not None, the number of seconds to put
383 between the last known Change and the
384 timestamp given to the -D argument. This
385 defaults to exactly half of the parent
386 Build's .treeStableTimer, but it could be
387 set to something else if your CVS change
388 notification has particularly weird
389 latency characteristics.
390
391 @type global_options: list of strings
392 @param global_options: these arguments are inserted in the cvs
393 command line, before the
394 'checkout'/'update' command word. See
395 'cvs --help-options' for a list of what
396 may be accepted here. ['-r'] will make
397 the checked out files read only. ['-r',
398 '-R'] will also assume the repository is
399 read-only (I assume this means it won't
400 use locks to insure atomic access to the
401 ,v files).
402
403 @type checkout_options: list of strings
404 @param checkout_options: these arguments are inserted in the cvs
405 command line, after 'checkout' but before
406 branch or revision specifiers.
407
408 @type export_options: list of strings
409 @param export_options: these arguments are inserted in the cvs
410 command line, after 'export' but before
411 branch or revision specifiers.
412
413 @type extra_options: list of strings
414 @param extra_options: these arguments are inserted in the cvs
415 command line, after 'checkout' or 'export' but before
416 branch or revision specifiers.
417 """
418
419 self.checkoutDelay = checkoutDelay
420 self.branch = branch
421 self.cvsroot = cvsroot
422
423 Source.__init__(self, **kwargs)
424 self.addFactoryArguments(cvsroot=cvsroot,
425 cvsmodule=cvsmodule,
426 global_options=global_options,
427 checkout_options=checkout_options,
428 export_options=export_options,
429 extra_options=extra_options,
430 branch=branch,
431 checkoutDelay=checkoutDelay,
432 login=login,
433 )
434
435 self.args.update({'cvsmodule': cvsmodule,
436 'global_options': global_options,
437 'checkout_options':checkout_options,
438 'export_options':export_options,
439 'extra_options':extra_options,
440 'login': login,
441 })
442
444 if not changes:
445 return None
446 lastChange = max([c.when for c in changes])
447 if self.checkoutDelay is not None:
448 when = lastChange + self.checkoutDelay
449 else:
450 lastSubmit = max([r.submittedAt for r in self.build.requests])
451 when = (lastChange + lastSubmit) / 2
452 return formatdate(when)
453
454 - def startVC(self, branch, revision, patch):
455 if self.slaveVersionIsOlderThan("cvs", "1.39"):
456
457
458
459
460
461 if (branch != self.branch
462 and self.args['mode'] in ("update", "copy")):
463 m = ("This buildslave (%s) does not know about multiple "
464 "branches, and using mode=%s would probably build the "
465 "wrong tree. "
466 "Refusing to build. Please upgrade the buildslave to "
467 "buildbot-0.7.0 or newer." % (self.build.slavename,
468 self.args['mode']))
469 log.msg(m)
470 raise BuildSlaveTooOldError(m)
471
472 if self.slaveVersionIsOlderThan("cvs", "2.10"):
473 if self.args['extra_options'] or self.args['export_options']:
474 m = ("This buildslave (%s) does not support export_options "
475 "or extra_options arguments to the CVS step."
476 % (self.build.slavename))
477 log.msg(m)
478 raise BuildSlaveTooOldError(m)
479
480
481 del self.args['export_options']
482 del self.args['extra_options']
483
484 if branch is None:
485 branch = "HEAD"
486 self.args['cvsroot'] = self.computeRepositoryURL(self.cvsroot)
487 self.args['branch'] = branch
488 self.args['revision'] = revision
489 self.args['patch'] = patch
490
491 if self.args['branch'] == "HEAD" and self.args['revision']:
492
493
494 self.args['branch'] = None
495
496
497 warnings = []
498 slavever = self.slaveVersion("cvs", "old")
499
500 if slavever == "old":
501
502 if self.args['mode'] == "export":
503 self.args['export'] = 1
504 elif self.args['mode'] == "clobber":
505 self.args['clobber'] = 1
506 elif self.args['mode'] == "copy":
507 self.args['copydir'] = "source"
508 self.args['tag'] = self.args['branch']
509 assert not self.args['patch']
510
511 cmd = LoggedRemoteCommand("cvs", self.args)
512 self.startCommand(cmd, warnings)
513
514
516 """I perform Subversion checkout/update operations."""
517
518 name = 'svn'
519 branch_placeholder = '%%BRANCH%%'
520
521 - def __init__(self, svnurl=None, baseURL=None, defaultBranch=None,
522 directory=None, username=None, password=None,
523 extra_args=None, keep_on_purge=None, ignore_ignores=None,
524 always_purge=None, depth=None, **kwargs):
525 """
526 @type svnurl: string
527 @param svnurl: the URL which points to the Subversion server,
528 combining the access method (HTTP, ssh, local file),
529 the repository host/port, the repository path, the
530 sub-tree within the repository, and the branch to
531 check out. Use exactly one of C{svnurl} and C{baseURL}.
532
533 @param baseURL: if branches are enabled, this is the base URL to
534 which a branch name will be appended. It should
535 probably end in a slash. Use exactly one of
536 C{svnurl} and C{baseURL}.
537
538 @param defaultBranch: if branches are enabled, this is the branch
539 to use if the Build does not specify one
540 explicitly. It will simply be appended
541 to C{baseURL} and the result handed to
542 the SVN command.
543
544 @type username: string
545 @param username: username to pass to svn's --username
546
547 @type password: string
548 @param password: password to pass to svn's --password
549 """
550
551 if not 'workdir' in kwargs and directory is not None:
552
553 warn("Please use workdir=, not directory=", DeprecationWarning)
554 kwargs['workdir'] = directory
555
556 self.svnurl = svnurl
557 self.baseURL = baseURL
558 self.branch = defaultBranch
559 self.username = username
560 self.password = password
561 self.extra_args = extra_args
562 self.keep_on_purge = keep_on_purge
563 self.ignore_ignores = ignore_ignores
564 self.always_purge = always_purge
565 self.depth = depth
566
567 Source.__init__(self, **kwargs)
568 self.addFactoryArguments(svnurl=svnurl,
569 baseURL=baseURL,
570 defaultBranch=defaultBranch,
571 directory=directory,
572 username=username,
573 password=password,
574 extra_args=extra_args,
575 keep_on_purge=keep_on_purge,
576 ignore_ignores=ignore_ignores,
577 always_purge=always_purge,
578 depth=depth,
579 )
580
581 if svnurl and baseURL:
582 raise ValueError("you must use either svnurl OR baseURL")
583
589
591 ''' Handle compatibility between old slaves/svn clients '''
592
593 slavever = self.slaveVersion("svn", "old")
594
595 if not slavever:
596 m = "slave does not have the 'svn' command"
597 raise BuildSlaveTooOldError(m)
598
599 if self.slaveVersionIsOlderThan("svn", "1.39"):
600
601
602
603
604
605 if (self.args['branch'] != self.branch
606 and self.args['mode'] in ("update", "copy")):
607 m = ("This buildslave (%s) does not know about multiple "
608 "branches, and using mode=%s would probably build the "
609 "wrong tree. "
610 "Refusing to build. Please upgrade the buildslave to "
611 "buildbot-0.7.0 or newer." % (self.build.slavename,
612 self.args['mode']))
613 raise BuildSlaveTooOldError(m)
614
615 if (self.depth is not None) and self.slaveVersionIsOlderThan("svn","2.9"):
616 m = ("This buildslave (%s) does not support svn depth "
617 "arguments. Refusing to build. "
618 "Please upgrade the buildslave." % (self.build.slavename))
619 raise BuildSlaveTooOldError(m)
620
621 if (self.username is not None or self.password is not None) \
622 and self.slaveVersionIsOlderThan("svn", "2.8"):
623 m = ("This buildslave (%s) does not support svn usernames "
624 "and passwords. "
625 "Refusing to build. Please upgrade the buildslave to "
626 "buildbot-0.7.10 or newer." % (self.build.slavename,))
627 raise BuildSlaveTooOldError(m)
628
629 - def getSvnUrl(self, branch, revision, patch):
630 ''' Compute the svn url that will be passed to the svn remote command '''
631 if self.svnurl:
632 return self.computeRepositoryURL(self.svnurl)
633 else:
634 if branch is None:
635 m = ("The SVN source step belonging to builder '%s' does not know "
636 "which branch to work with. This means that the change source "
637 "did not specify a branch and that defaultBranch is None." \
638 % self.build.builder.name)
639 raise RuntimeError(m)
640
641 computed = self.computeRepositoryURL(self.baseURL)
642
643 if self.branch_placeholder in self.baseURL:
644 return computed.replace(self.branch_placeholder, branch)
645 else:
646 return computed + branch
647
648 - def startVC(self, branch, revision, patch):
649 warnings = []
650
651 self.checkCompatibility()
652
653 self.args['svnurl'] = self.getSvnUrl(branch, revision, patch)
654 self.args['revision'] = revision
655 self.args['patch'] = patch
656 self.args['always_purge'] = self.always_purge
657
658
659 if self.depth is not None:
660 self.args['depth'] = self.depth
661
662 if self.username is not None:
663 self.args['username'] = self.username
664 if self.password is not None:
665 self.args['password'] = self.password
666
667 if self.extra_args is not None:
668 self.args['extra_args'] = self.extra_args
669
670 revstuff = []
671
672 if self.args['svnurl'].find('trunk') == -1:
673 revstuff.append("[branch]")
674 if revision is not None:
675 revstuff.append("r%s" % revision)
676 if patch is not None:
677 revstuff.append("[patch]")
678 self.description.extend(revstuff)
679 self.descriptionDone.extend(revstuff)
680
681 cmd = LoggedRemoteCommand("svn", self.args)
682 self.startCommand(cmd, warnings)
683
684
686 """Check out a source tree from a Darcs repository at 'repourl'.
687
688 Darcs has no concept of file modes. This means the eXecute-bit will be
689 cleared on all source files. As a result, you may need to invoke
690 configuration scripts with something like:
691
692 C{s(step.Configure, command=['/bin/sh', './configure'])}
693 """
694
695 name = "darcs"
696
697 - def __init__(self, repourl=None, baseURL=None, defaultBranch=None,
698 **kwargs):
699 """
700 @type repourl: string
701 @param repourl: the URL which points at the Darcs repository. This
702 is used as the default branch. Using C{repourl} does
703 not enable builds of alternate branches: use
704 C{baseURL} to enable this. Use either C{repourl} or
705 C{baseURL}, not both.
706
707 @param baseURL: if branches are enabled, this is the base URL to
708 which a branch name will be appended. It should
709 probably end in a slash. Use exactly one of
710 C{repourl} and C{baseURL}.
711
712 @param defaultBranch: if branches are enabled, this is the branch
713 to use if the Build does not specify one
714 explicitly. It will simply be appended to
715 C{baseURL} and the result handed to the
716 'darcs pull' command.
717 """
718 self.repourl = repourl
719 self.baseURL = baseURL
720 self.branch = defaultBranch
721 Source.__init__(self, **kwargs)
722 self.addFactoryArguments(repourl=repourl,
723 baseURL=baseURL,
724 defaultBranch=defaultBranch,
725 )
726 assert self.args['mode'] != "export", \
727 "Darcs does not have an 'export' mode"
728 if repourl and baseURL:
729 raise ValueError("you must provide exactly one of repourl and"
730 " baseURL")
731
732 - def startVC(self, branch, revision, patch):
775
776
778 """Check out a source tree from a git repository 'repourl'."""
779
780 name = "git"
781
782 - def __init__(self, repourl=None,
783 branch="master",
784 submodules=False,
785 ignore_ignores=None,
786 reference=None,
787 shallow=False,
788 progress=False,
789 **kwargs):
790 """
791 @type repourl: string
792 @param repourl: the URL which points at the git repository
793
794 @type branch: string
795 @param branch: The branch or tag to check out by default. If
796 a build specifies a different branch, it will
797 be used instead of this.
798
799 @type submodules: boolean
800 @param submodules: Whether or not to update (and initialize)
801 git submodules.
802
803 @type reference: string
804 @param reference: The path to a reference repository to obtain
805 objects from, if any.
806
807 @type shallow: boolean
808 @param shallow: Use a shallow or clone, if possible
809
810 @type progress: boolean
811 @param progress: Pass the --progress option when fetching. This
812 can solve long fetches getting killed due to
813 lack of output, but requires Git 1.7.2+.
814 """
815 Source.__init__(self, **kwargs)
816 self.repourl = repourl
817 self.addFactoryArguments(repourl=repourl,
818 branch=branch,
819 submodules=submodules,
820 ignore_ignores=ignore_ignores,
821 reference=reference,
822 shallow=shallow,
823 progress=progress,
824 )
825 self.args.update({'branch': branch,
826 'submodules': submodules,
827 'ignore_ignores': ignore_ignores,
828 'reference': reference,
829 'shallow': shallow,
830 'progress': progress,
831 })
832
837
838 - def startVC(self, branch, revision, patch):
839 if branch is not None:
840 self.args['branch'] = branch
841
842 self.args['repourl'] = self.computeRepositoryURL(self.repourl)
843 self.args['revision'] = revision
844 self.args['patch'] = patch
845
846
847 try:
848
849 self.args['gerrit_branch'] = self.build.getProperty("event.patchSet.ref")
850 self.setProperty("gerrit_branch", self.args['gerrit_branch'])
851 except KeyError:
852 try:
853
854 change = self.build.getProperty("gerrit_change").split('/')
855 if len(change) == 2:
856 self.args['gerrit_branch'] = "refs/changes/%2.2d/%d/%d" \
857 % (int(change[0]) % 100, int(change[0]), int(change[1]))
858 self.setProperty("gerrit_branch", self.args['gerrit_branch'])
859 except:
860 pass
861
862 slavever = self.slaveVersion("git")
863 if not slavever:
864 raise BuildSlaveTooOldError("slave is too old, does not know "
865 "about git")
866 cmd = LoggedRemoteCommand("git", self.args)
867 self.startCommand(cmd)
868
869
871 """Check out a source tree from a repo repository described by manifest."""
872
873 name = "repo"
874
875 - def __init__(self,
876 manifest_url=None,
877 manifest_branch="master",
878 manifest_file="default.xml",
879 tarball=None,
880 **kwargs):
881 """
882 @type manifest_url: string
883 @param manifest_url: The URL which points at the repo manifests repository.
884
885 @type manifest_branch: string
886 @param manifest_branch: The manifest branch to check out by default.
887
888 @type manifest_file: string
889 @param manifest_file: The manifest to use for sync.
890
891 """
892 Source.__init__(self, **kwargs)
893 self.manifest_url = manifest_url
894 self.addFactoryArguments(manifest_url=manifest_url,
895 manifest_branch=manifest_branch,
896 manifest_file=manifest_file,
897 tarball=tarball,
898 )
899 self.args.update({'manifest_branch': manifest_branch,
900 'manifest_file': manifest_file,
901 'tarball': tarball,
902 })
903
908
910 """
911 lets try to be nice in the format we want
912 can support several instances of "repo download proj number/patch" (direct copy paste from gerrit web site)
913 or several instances of "proj number/patch" (simpler version)
914 This feature allows integrator to build with several pending interdependant changes.
915 returns list of repo downloads sent to the buildslave
916 """
917 import re
918 if s == None:
919 return []
920 re1 = re.compile("repo download ([^ ]+) ([0-9]+/[0-9]+)")
921 re2 = re.compile("([^ ]+) ([0-9]+/[0-9]+)")
922 re3 = re.compile("([^ ]+)/([0-9]+/[0-9]+)")
923 ret = []
924 for cur_re in [re1, re2, re3]:
925 res = cur_re.search(s)
926 while res:
927 ret.append("%s %s" % (res.group(1), res.group(2)))
928 s = s[:res.start(0)] + s[res.end(0):]
929 res = cur_re.search(s)
930 return ret
931
932 - def startVC(self, branch, revision, patch):
933 self.args['manifest_url'] = self.computeRepositoryURL(self.manifest_url)
934
935
936 downloads = []
937
938
939 for change in self.build.allChanges():
940 if (change.properties.has_key("event.type") and
941 change.properties["event.type"] == "patchset-created"):
942 downloads.append("%s %s/%s"% (change.properties["event.change.project"],
943 change.properties["event.change.number"],
944 change.properties["event.patchSet.number"]))
945
946
947
948
949 for propName in ["repo_d"] + ["repo_d%d" % i for i in xrange(0,10)] + \
950 ["repo_download"] + ["repo_download%d" % i for i in xrange(0,10)]:
951 try:
952 s = self.build.getProperty(propName)
953 downloads.extend(self.parseDownloadProperty(s))
954 except KeyError:
955 pass
956
957 if downloads:
958 self.args["repo_downloads"] = downloads
959 self.setProperty("repo_downloads", downloads)
960
961 slavever = self.slaveVersion("repo")
962 if not slavever:
963 raise BuildSlaveTooOldError("slave is too old, does not know "
964 "about repo")
965 cmd = LoggedRemoteCommand("repo", self.args)
966 self.startCommand(cmd)
967
969 if cmd.updates.has_key("repo_downloaded"):
970 repo_downloaded = cmd.updates["repo_downloaded"][-1]
971 if repo_downloaded:
972 self.setProperty("repo_downloaded", str(repo_downloaded), "Source")
973
974
976 """Check out a source tree from a bzr (Bazaar) repository at 'repourl'.
977
978 """
979
980 name = "bzr"
981
982 - def __init__(self, repourl=None, baseURL=None, defaultBranch=None,
983 forceSharedRepo=None,
984 **kwargs):
985 """
986 @type repourl: string
987 @param repourl: the URL which points at the bzr repository. This
988 is used as the default branch. Using C{repourl} does
989 not enable builds of alternate branches: use
990 C{baseURL} to enable this. Use either C{repourl} or
991 C{baseURL}, not both.
992
993 @param baseURL: if branches are enabled, this is the base URL to
994 which a branch name will be appended. It should
995 probably end in a slash. Use exactly one of
996 C{repourl} and C{baseURL}.
997
998 @param defaultBranch: if branches are enabled, this is the branch
999 to use if the Build does not specify one
1000 explicitly. It will simply be appended to
1001 C{baseURL} and the result handed to the
1002 'bzr checkout pull' command.
1003
1004
1005 @param forceSharedRepo: Boolean, defaults to False. If set to True,
1006 the working directory will be made into a
1007 bzr shared repository if it is not already.
1008 Shared repository greatly reduces the amount
1009 of history data that needs to be downloaded
1010 if not using update/copy mode, or if using
1011 update/copy mode with multiple branches.
1012 """
1013 self.repourl = repourl
1014 self.baseURL = baseURL
1015 self.branch = defaultBranch
1016 Source.__init__(self, **kwargs)
1017 self.addFactoryArguments(repourl=repourl,
1018 baseURL=baseURL,
1019 defaultBranch=defaultBranch,
1020 forceSharedRepo=forceSharedRepo
1021 )
1022 self.args.update({'forceSharedRepo': forceSharedRepo})
1023 if repourl and baseURL:
1024 raise ValueError("you must provide exactly one of repourl and"
1025 " baseURL")
1026
1028 if not changes:
1029 return None
1030 lastChange = max([int(c.revision) for c in changes])
1031 return lastChange
1032
1033 - def startVC(self, branch, revision, patch):
1057
1058
1060 """Check out a source tree from a mercurial repository 'repourl'."""
1061
1062 name = "hg"
1063
1064 - def __init__(self, repourl=None, baseURL=None, defaultBranch=None,
1065 branchType='dirname', clobberOnBranchChange=True, **kwargs):
1066 """
1067 @type repourl: string
1068 @param repourl: the URL which points at the Mercurial repository.
1069 This uses the 'default' branch unless defaultBranch is
1070 specified below and the C{branchType} is set to
1071 'inrepo'. It is an error to specify a branch without
1072 setting the C{branchType} to 'inrepo'.
1073
1074 @param baseURL: if 'dirname' branches are enabled, this is the base URL
1075 to which a branch name will be appended. It should
1076 probably end in a slash. Use exactly one of C{repourl}
1077 and C{baseURL}.
1078
1079 @param defaultBranch: if branches are enabled, this is the branch
1080 to use if the Build does not specify one
1081 explicitly.
1082 For 'dirname' branches, It will simply be
1083 appended to C{baseURL} and the result handed to
1084 the 'hg update' command.
1085 For 'inrepo' branches, this specifies the named
1086 revision to which the tree will update after a
1087 clone.
1088
1089 @param branchType: either 'dirname' or 'inrepo' depending on whether
1090 the branch name should be appended to the C{baseURL}
1091 or the branch is a mercurial named branch and can be
1092 found within the C{repourl}
1093
1094 @param clobberOnBranchChange: boolean, defaults to True. If set and
1095 using inrepos branches, clobber the tree
1096 at each branch change. Otherwise, just
1097 update to the branch.
1098 """
1099 self.repourl = repourl
1100 self.baseURL = baseURL
1101 self.branch = defaultBranch
1102 self.branchType = branchType
1103 self.clobberOnBranchChange = clobberOnBranchChange
1104 Source.__init__(self, **kwargs)
1105 self.addFactoryArguments(repourl=repourl,
1106 baseURL=baseURL,
1107 defaultBranch=defaultBranch,
1108 branchType=branchType,
1109 clobberOnBranchChange=clobberOnBranchChange,
1110 )
1111 if repourl and baseURL:
1112 raise ValueError("you must provide exactly one of repourl and"
1113 " baseURL")
1114
1115 - def startVC(self, branch, revision, patch):
1142
1144 if not changes:
1145 return None
1146
1147
1148
1149
1150 if len(changes) > 1:
1151 log.msg("Mercurial.computeSourceRevision: warning: "
1152 "there are %d changes here, assuming the last one is "
1153 "the most recent" % len(changes))
1154 return changes[-1].revision
1155
1156
1158 """ P4 is a class for accessing perforce revision control"""
1159 name = "p4"
1160
1161 - def __init__(self, p4base=None, defaultBranch=None, p4port=None, p4user=None,
1162 p4passwd=None, p4extra_views=[], p4line_end='local',
1163 p4client='buildbot_%(slave)s_%(builder)s', **kwargs):
1164 """
1165 @type p4base: string
1166 @param p4base: A view into a perforce depot, typically
1167 "//depot/proj/"
1168
1169 @type defaultBranch: string
1170 @param defaultBranch: Identify a branch to build by default. Perforce
1171 is a view based branching system. So, the branch
1172 is normally the name after the base. For example,
1173 branch=1.0 is view=//depot/proj/1.0/...
1174 branch=1.1 is view=//depot/proj/1.1/...
1175
1176 @type p4port: string
1177 @param p4port: Specify the perforce server to connection in the format
1178 <host>:<port>. Example "perforce.example.com:1666"
1179
1180 @type p4user: string
1181 @param p4user: The perforce user to run the command as.
1182
1183 @type p4passwd: string
1184 @param p4passwd: The password for the perforce user.
1185
1186 @type p4extra_views: list of tuples
1187 @param p4extra_views: Extra views to be added to
1188 the client that is being used.
1189
1190 @type p4line_end: string
1191 @param p4line_end: value of the LineEnd client specification property
1192
1193 @type p4client: string
1194 @param p4client: The perforce client to use for this buildslave.
1195 """
1196
1197 self.p4base = p4base
1198 self.branch = defaultBranch
1199 Source.__init__(self, **kwargs)
1200 self.addFactoryArguments(p4base=p4base,
1201 defaultBranch=defaultBranch,
1202 p4port=p4port,
1203 p4user=p4user,
1204 p4passwd=p4passwd,
1205 p4extra_views=p4extra_views,
1206 p4line_end=p4line_end,
1207 p4client=p4client,
1208 )
1209 self.args['p4port'] = p4port
1210 self.args['p4user'] = p4user
1211 self.args['p4passwd'] = p4passwd
1212 self.args['p4extra_views'] = p4extra_views
1213 self.args['p4line_end'] = p4line_end
1214 self.p4client = p4client
1215
1223
1225 if not changes:
1226 return None
1227 lastChange = max([int(c.revision) for c in changes])
1228 return lastChange
1229
1230 - def startVC(self, branch, revision, patch):
1239
1241 """This is a partial solution for using a P4 source repository. You are
1242 required to manually set up each build slave with a useful P4
1243 environment, which means setting various per-slave environment variables,
1244 and creating a P4 client specification which maps the right files into
1245 the slave's working directory. Once you have done that, this step merely
1246 performs a 'p4 sync' to update that workspace with the newest files.
1247
1248 Each slave needs the following environment:
1249
1250 - PATH: the 'p4' binary must be on the slave's PATH
1251 - P4USER: each slave needs a distinct user account
1252 - P4CLIENT: each slave needs a distinct client specification
1253
1254 You should use 'p4 client' (?) to set up a client view spec which maps
1255 the desired files into $SLAVEBASE/$BUILDERBASE/source .
1256 """
1257
1258 name = "p4sync"
1259
1260 - def __init__(self, p4port, p4user, p4passwd, p4client, **kwargs):
1261 assert kwargs['mode'] == "copy", "P4Sync can only be used in mode=copy"
1262 self.branch = None
1263 Source.__init__(self, **kwargs)
1264 self.addFactoryArguments(p4port=p4port,
1265 p4user=p4user,
1266 p4passwd=p4passwd,
1267 p4client=p4client,
1268 )
1269 self.args['p4port'] = p4port
1270 self.args['p4user'] = p4user
1271 self.args['p4passwd'] = p4passwd
1272 self.args['p4client'] = p4client
1273
1275 if not changes:
1276 return None
1277 lastChange = max([int(c.revision) for c in changes])
1278 return lastChange
1279
1280 - def startVC(self, branch, revision, patch):
1285