1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 from warnings import warn
18 from email.Utils import formatdate
19 from twisted.python import log
20 from twisted.internet import defer
21 from zope.interface import implements
22 from buildbot.process.buildstep import LoggingBuildStep, LoggedRemoteCommand
23 from buildbot.interfaces import BuildSlaveTooOldError, IRenderable
24 from buildbot.status.builder import SKIPPED
25 from buildbot.status.results import FAILURE
26
60
61 -class Source(LoggingBuildStep):
62 """This is a base class to generate a source tree in the buildslave.
63 Each version control system has a specialized subclass, and is expected
64 to override __init__ and implement computeSourceRevision() and
65 startVC(). The class as a whole builds up the self.args dictionary, then
66 starts a LoggedRemoteCommand with those arguments.
67 """
68
69 renderables = [ 'workdir' ]
70
71
72 haltOnFailure = True
73 flunkOnFailure = True
74 notReally = False
75
76 branch = None
77
78 - def __init__(self, workdir=None, mode='update', alwaysUseLatest=False,
79 timeout=20*60, retry=None, logEnviron=True, **kwargs):
80 """
81 @type workdir: string
82 @param workdir: local directory (relative to the Builder's root)
83 where the tree should be placed
84
85 @type mode: string
86 @param mode: the kind of VC operation that is desired:
87 - 'update': specifies that the checkout/update should be
88 performed directly into the workdir. Each build is performed
89 in the same directory, allowing for incremental builds. This
90 minimizes disk space, bandwidth, and CPU time. However, it
91 may encounter problems if the build process does not handle
92 dependencies properly (if you must sometimes do a 'clean
93 build' to make sure everything gets compiled), or if source
94 files are deleted but generated files can influence test
95 behavior (e.g. python's .pyc files), or when source
96 directories are deleted but generated files prevent CVS from
97 removing them. When used with a patched checkout, from a
98 previous buildbot try for instance, it will try to "revert"
99 the changes first and will do a clobber if it is unable to
100 get a clean checkout. The behavior is SCM-dependent.
101
102 - 'copy': specifies that the source-controlled workspace
103 should be maintained in a separate directory (called the
104 'copydir'), using checkout or update as necessary. For each
105 build, a new workdir is created with a copy of the source
106 tree (rm -rf workdir; cp -R -P -p copydir workdir). This
107 doubles the disk space required, but keeps the bandwidth low
108 (update instead of a full checkout). A full 'clean' build
109 is performed each time. This avoids any generated-file
110 build problems, but is still occasionally vulnerable to
111 problems such as a CVS repository being manually rearranged
112 (causing CVS errors on update) which are not an issue with
113 a full checkout.
114
115 - 'clobber': specifies that the working directory should be
116 deleted each time, necessitating a full checkout for each
117 build. This insures a clean build off a complete checkout,
118 avoiding any of the problems described above, but is
119 bandwidth intensive, as the whole source tree must be
120 pulled down for each build.
121
122 - 'export': is like 'clobber', except that e.g. the 'cvs
123 export' command is used to create the working directory.
124 This command removes all VC metadata files (the
125 CVS/.svn/{arch} directories) from the tree, which is
126 sometimes useful for creating source tarballs (to avoid
127 including the metadata in the tar file). Not all VC systems
128 support export.
129
130 @type alwaysUseLatest: boolean
131 @param alwaysUseLatest: whether to always update to the most
132 recent available sources for this build.
133
134 Normally the Source step asks its Build for a list of all
135 Changes that are supposed to go into the build, then computes a
136 'source stamp' (revision number or timestamp) that will cause
137 exactly that set of changes to be present in the checked out
138 tree. This is turned into, e.g., 'cvs update -D timestamp', or
139 'svn update -r revnum'. If alwaysUseLatest=True, bypass this
140 computation and always update to the latest available sources
141 for each build.
142
143 The source stamp helps avoid a race condition in which someone
144 commits a change after the master has decided to start a build
145 but before the slave finishes checking out the sources. At best
146 this results in a build which contains more changes than the
147 buildmaster thinks it has (possibly resulting in the wrong
148 person taking the blame for any problems that result), at worst
149 is can result in an incoherent set of sources (splitting a
150 non-atomic commit) which may not build at all.
151
152 @type retry: tuple of ints (delay, repeats) (or None)
153 @param retry: if provided, VC update failures are re-attempted up
154 to REPEATS times, with DELAY seconds between each
155 attempt. Some users have slaves with poor connectivity
156 to their VC repository, and they say that up to 80% of
157 their build failures are due to transient network
158 failures that could be handled by simply retrying a
159 couple times.
160
161 @type logEnviron: boolean
162 @param logEnviron: If this option is true (the default), then the
163 step's logfile will describe the environment
164 variables on the slave. In situations where the
165 environment is not relevant and is long, it may
166 be easier to set logEnviron=False.
167
168 """
169
170 LoggingBuildStep.__init__(self, **kwargs)
171 self.addFactoryArguments(workdir=workdir,
172 mode=mode,
173 alwaysUseLatest=alwaysUseLatest,
174 timeout=timeout,
175 retry=retry,
176 )
177
178 assert mode in ("update", "copy", "clobber", "export")
179 if retry:
180 delay, repeats = retry
181 assert isinstance(repeats, int)
182 assert repeats > 0
183 self.args = {'mode': mode,
184 'timeout': timeout,
185 'retry': retry,
186 'patch': None,
187 }
188
189 self.workdir = workdir
190
191 self.alwaysUseLatest = alwaysUseLatest
192
193 self.logEnviron = logEnviron
194
195
196 description = ["updating"]
197 descriptionDone = ["update"]
198 if mode == "clobber":
199 description = ["checkout"]
200
201 descriptionDone = ["checkout"]
202 elif mode == "export":
203 description = ["exporting"]
204 descriptionDone = ["export"]
205 self.description = description
206 self.descriptionDone = descriptionDone
207
210
213
218
220 """Each subclass must implement this method to do something more
221 precise than -rHEAD every time. For version control systems that use
222 repository-wide change numbers (SVN, P4), this can simply take the
223 maximum such number from all the changes involved in this build. For
224 systems that do not (CVS), it needs to create a timestamp based upon
225 the latest Change, the Build's treeStableTimer, and an optional
226 self.checkoutDelay value."""
227 return None
228
267
269 if cmd.updates.has_key("got_revision"):
270 got_revision = cmd.updates["got_revision"][-1]
271 if got_revision is not None:
272 self.setProperty("got_revision", str(got_revision), "Source")
273
274
275
277 """I perform BitKeeper checkout/update operations."""
278
279 name = 'bk'
280
281 renderables = [ 'bkurl', 'baseURL' ]
282
283 - def __init__(self, bkurl=None, baseURL=None,
284 directory=None, extra_args=None, **kwargs):
285 """
286 @type bkurl: string
287 @param bkurl: the URL which points to the BitKeeper server.
288
289 @type baseURL: string
290 @param baseURL: if branches are enabled, this is the base URL to
291 which a branch name will be appended. It should
292 probably end in a slash. Use exactly one of
293 C{bkurl} and C{baseURL}.
294 """
295
296 self.bkurl = _ComputeRepositoryURL(bkurl)
297 self.baseURL = _ComputeRepositoryURL(baseURL)
298 self.extra_args = extra_args
299
300 Source.__init__(self, **kwargs)
301 self.addFactoryArguments(bkurl=bkurl,
302 baseURL=baseURL,
303 directory=directory,
304 extra_args=extra_args,
305 )
306
307 if bkurl and baseURL:
308 raise ValueError("you must use exactly one of bkurl and baseURL")
309
310
313
314
315 - def startVC(self, branch, revision, patch):
345
346
347
349 """I do CVS checkout/update operations.
350
351 Note: if you are doing anonymous/pserver CVS operations, you will need
352 to manually do a 'cvs login' on each buildslave before the slave has any
353 hope of success. XXX: fix then, take a cvs password as an argument and
354 figure out how to do a 'cvs login' on each build
355 """
356
357 name = "cvs"
358
359 renderables = [ "cvsroot" ]
360
361
362
363
364
365
366
367
368
369
370
371 - def __init__(self, cvsroot=None, cvsmodule="",
372 global_options=[], branch=None, checkoutDelay=None,
373 checkout_options=[], export_options=[], extra_options=[],
374 login=None,
375 **kwargs):
376
377 """
378 @type cvsroot: string
379 @param cvsroot: CVS Repository from which the source tree should
380 be obtained. '/home/warner/Repository' for local
381 or NFS-reachable repositories,
382 ':pserver:anon@foo.com:/cvs' for anonymous CVS,
383 'user@host.com:/cvs' for non-anonymous CVS or
384 CVS over ssh. Lots of possibilities, check the
385 CVS documentation for more.
386
387 @type cvsmodule: string
388 @param cvsmodule: subdirectory of CVS repository that should be
389 retrieved
390
391 @type login: string or None
392 @param login: if not None, a string which will be provided as a
393 password to the 'cvs login' command, used when a
394 :pserver: method is used to access the repository.
395 This login is only needed once, but must be run
396 each time (just before the CVS operation) because
397 there is no way for the buildslave to tell whether
398 it was previously performed or not.
399
400 @type branch: string
401 @param branch: the default branch name, will be used in a '-r'
402 argument to specify which branch of the source tree
403 should be used for this checkout. Defaults to None,
404 which means to use 'HEAD'.
405
406 @type checkoutDelay: int or None
407 @param checkoutDelay: if not None, the number of seconds to put
408 between the last known Change and the
409 timestamp given to the -D argument. This
410 defaults to exactly half of the parent
411 Build's .treeStableTimer, but it could be
412 set to something else if your CVS change
413 notification has particularly weird
414 latency characteristics.
415
416 @type global_options: list of strings
417 @param global_options: these arguments are inserted in the cvs
418 command line, before the
419 'checkout'/'update' command word. See
420 'cvs --help-options' for a list of what
421 may be accepted here. ['-r'] will make
422 the checked out files read only. ['-r',
423 '-R'] will also assume the repository is
424 read-only (I assume this means it won't
425 use locks to insure atomic access to the
426 ,v files).
427
428 @type checkout_options: list of strings
429 @param checkout_options: these arguments are inserted in the cvs
430 command line, after 'checkout' but before
431 branch or revision specifiers.
432
433 @type export_options: list of strings
434 @param export_options: these arguments are inserted in the cvs
435 command line, after 'export' but before
436 branch or revision specifiers.
437
438 @type extra_options: list of strings
439 @param extra_options: these arguments are inserted in the cvs
440 command line, after 'checkout' or 'export' but before
441 branch or revision specifiers.
442 """
443
444 self.checkoutDelay = checkoutDelay
445 self.branch = branch
446 self.cvsroot = _ComputeRepositoryURL(cvsroot)
447
448 Source.__init__(self, **kwargs)
449 self.addFactoryArguments(cvsroot=cvsroot,
450 cvsmodule=cvsmodule,
451 global_options=global_options,
452 checkout_options=checkout_options,
453 export_options=export_options,
454 extra_options=extra_options,
455 branch=branch,
456 checkoutDelay=checkoutDelay,
457 login=login,
458 )
459
460 self.args.update({'cvsmodule': cvsmodule,
461 'global_options': global_options,
462 'checkout_options':checkout_options,
463 'export_options':export_options,
464 'extra_options':extra_options,
465 'login': login,
466 })
467
469 if not changes:
470 return None
471 lastChange = max([c.when for c in changes])
472 if self.checkoutDelay is not None:
473 when = lastChange + self.checkoutDelay
474 else:
475 lastSubmit = max([br.submittedAt for br in self.build.requests])
476 when = (lastChange + lastSubmit) / 2
477 return formatdate(when)
478
479 - def startVC(self, branch, revision, patch):
480 if self.slaveVersionIsOlderThan("cvs", "1.39"):
481
482
483
484
485
486 if (branch != self.branch
487 and self.args['mode'] in ("update", "copy")):
488 m = ("This buildslave (%s) does not know about multiple "
489 "branches, and using mode=%s would probably build the "
490 "wrong tree. "
491 "Refusing to build. Please upgrade the buildslave to "
492 "buildbot-0.7.0 or newer." % (self.build.slavename,
493 self.args['mode']))
494 log.msg(m)
495 raise BuildSlaveTooOldError(m)
496
497 if self.slaveVersionIsOlderThan("cvs", "2.10"):
498 if self.args['extra_options'] or self.args['export_options']:
499 m = ("This buildslave (%s) does not support export_options "
500 "or extra_options arguments to the CVS step."
501 % (self.build.slavename))
502 log.msg(m)
503 raise BuildSlaveTooOldError(m)
504
505
506 del self.args['export_options']
507 del self.args['extra_options']
508
509 if branch is None:
510 branch = "HEAD"
511 self.args['cvsroot'] = self.cvsroot
512 self.args['branch'] = branch
513 self.args['revision'] = revision
514 self.args['patch'] = patch
515
516 if self.args['branch'] == "HEAD" and self.args['revision']:
517
518
519 self.args['branch'] = None
520
521
522 warnings = []
523 slavever = self.slaveVersion("cvs", "old")
524
525 if slavever == "old":
526
527 if self.args['mode'] == "export":
528 self.args['export'] = 1
529 elif self.args['mode'] == "clobber":
530 self.args['clobber'] = 1
531 elif self.args['mode'] == "copy":
532 self.args['copydir'] = "source"
533 self.args['tag'] = self.args['branch']
534 assert not self.args['patch']
535
536 cmd = LoggedRemoteCommand("cvs", self.args)
537 self.startCommand(cmd, warnings)
538
539
541 """I perform Subversion checkout/update operations."""
542
543 name = 'svn'
544 branch_placeholder = '%%BRANCH%%'
545
546 renderables = [ 'svnurl', 'baseURL' ]
547
548 - def __init__(self, svnurl=None, baseURL=None, defaultBranch=None,
549 directory=None, username=None, password=None,
550 extra_args=None, keep_on_purge=None, ignore_ignores=None,
551 always_purge=None, depth=None, **kwargs):
552 """
553 @type svnurl: string
554 @param svnurl: the URL which points to the Subversion server,
555 combining the access method (HTTP, ssh, local file),
556 the repository host/port, the repository path, the
557 sub-tree within the repository, and the branch to
558 check out. Use exactly one of C{svnurl} and C{baseURL}.
559
560 @param baseURL: if branches are enabled, this is the base URL to
561 which a branch name will be appended. It should
562 probably end in a slash. Use exactly one of
563 C{svnurl} and C{baseURL}.
564
565 @param defaultBranch: if branches are enabled, this is the branch
566 to use if the Build does not specify one
567 explicitly. It will simply be appended
568 to C{baseURL} and the result handed to
569 the SVN command.
570
571 @type username: string
572 @param username: username to pass to svn's --username
573
574 @type password: string
575 @param password: password to pass to svn's --password
576 """
577
578 if not 'workdir' in kwargs and directory is not None:
579
580 warn("Please use workdir=, not directory=", DeprecationWarning)
581 kwargs['workdir'] = directory
582
583 self.svnurl = svnurl and _ComputeRepositoryURL(svnurl)
584 self.baseURL = _ComputeRepositoryURL(baseURL)
585 self.branch = defaultBranch
586 self.username = username
587 self.password = password
588 self.extra_args = extra_args
589 self.keep_on_purge = keep_on_purge
590 self.ignore_ignores = ignore_ignores
591 self.always_purge = always_purge
592 self.depth = depth
593
594 Source.__init__(self, **kwargs)
595 self.addFactoryArguments(svnurl=svnurl,
596 baseURL=baseURL,
597 defaultBranch=defaultBranch,
598 directory=directory,
599 username=username,
600 password=password,
601 extra_args=extra_args,
602 keep_on_purge=keep_on_purge,
603 ignore_ignores=ignore_ignores,
604 always_purge=always_purge,
605 depth=depth,
606 )
607
608 if svnurl and baseURL:
609 raise ValueError("you must use either svnurl OR baseURL")
610
616
618 ''' Handle compatibility between old slaves/svn clients '''
619
620 slavever = self.slaveVersion("svn", "old")
621
622 if not slavever:
623 m = "slave does not have the 'svn' command"
624 raise BuildSlaveTooOldError(m)
625
626 if self.slaveVersionIsOlderThan("svn", "1.39"):
627
628
629
630
631
632 if (self.args['branch'] != self.branch
633 and self.args['mode'] in ("update", "copy")):
634 m = ("This buildslave (%s) does not know about multiple "
635 "branches, and using mode=%s would probably build the "
636 "wrong tree. "
637 "Refusing to build. Please upgrade the buildslave to "
638 "buildbot-0.7.0 or newer." % (self.build.slavename,
639 self.args['mode']))
640 raise BuildSlaveTooOldError(m)
641
642 if (self.depth is not None) and self.slaveVersionIsOlderThan("svn","2.9"):
643 m = ("This buildslave (%s) does not support svn depth "
644 "arguments. Refusing to build. "
645 "Please upgrade the buildslave." % (self.build.slavename))
646 raise BuildSlaveTooOldError(m)
647
648 if (self.username is not None or self.password is not None) \
649 and self.slaveVersionIsOlderThan("svn", "2.8"):
650 m = ("This buildslave (%s) does not support svn usernames "
651 "and passwords. "
652 "Refusing to build. Please upgrade the buildslave to "
653 "buildbot-0.7.10 or newer." % (self.build.slavename,))
654 raise BuildSlaveTooOldError(m)
655
657 ''' Compute the svn url that will be passed to the svn remote command '''
658 if self.svnurl:
659 return self.svnurl
660 else:
661 if branch is None:
662 m = ("The SVN source step belonging to builder '%s' does not know "
663 "which branch to work with. This means that the change source "
664 "did not specify a branch and that defaultBranch is None." \
665 % self.build.builder.name)
666 raise RuntimeError(m)
667
668 computed = self.baseURL
669
670 if self.branch_placeholder in self.baseURL:
671 return computed.replace(self.branch_placeholder, branch)
672 else:
673 return computed + branch
674
675 - def startVC(self, branch, revision, patch):
676 warnings = []
677
678 self.checkCompatibility()
679
680 self.args['svnurl'] = self.getSvnUrl(branch)
681 self.args['revision'] = revision
682 self.args['patch'] = patch
683 self.args['always_purge'] = self.always_purge
684
685
686 if self.depth is not None:
687 self.args['depth'] = self.depth
688
689 if self.username is not None:
690 self.args['username'] = self.username
691 if self.password is not None:
692 self.args['password'] = self.password
693
694 if self.extra_args is not None:
695 self.args['extra_args'] = self.extra_args
696
697 revstuff = []
698
699 if self.args['svnurl'].find('trunk') == -1:
700 revstuff.append("[branch]")
701 if revision is not None:
702 revstuff.append("r%s" % revision)
703 if patch is not None:
704 revstuff.append("[patch]")
705 self.description.extend(revstuff)
706 self.descriptionDone.extend(revstuff)
707
708 cmd = LoggedRemoteCommand("svn", self.args)
709 self.startCommand(cmd, warnings)
710
711
713 """Check out a source tree from a Darcs repository at 'repourl'.
714
715 Darcs has no concept of file modes. This means the eXecute-bit will be
716 cleared on all source files. As a result, you may need to invoke
717 configuration scripts with something like:
718
719 C{s(step.Configure, command=['/bin/sh', './configure'])}
720 """
721
722 name = "darcs"
723
724 renderables = [ 'repourl', 'baseURL' ]
725
726 - def __init__(self, repourl=None, baseURL=None, defaultBranch=None,
727 **kwargs):
728 """
729 @type repourl: string
730 @param repourl: the URL which points at the Darcs repository. This
731 is used as the default branch. Using C{repourl} does
732 not enable builds of alternate branches: use
733 C{baseURL} to enable this. Use either C{repourl} or
734 C{baseURL}, not both.
735
736 @param baseURL: if branches are enabled, this is the base URL to
737 which a branch name will be appended. It should
738 probably end in a slash. Use exactly one of
739 C{repourl} and C{baseURL}.
740
741 @param defaultBranch: if branches are enabled, this is the branch
742 to use if the Build does not specify one
743 explicitly. It will simply be appended to
744 C{baseURL} and the result handed to the
745 'darcs pull' command.
746 """
747 self.repourl = _ComputeRepositoryURL(repourl)
748 self.baseURL = _ComputeRepositoryURL(baseURL)
749 self.branch = defaultBranch
750 Source.__init__(self, **kwargs)
751 self.addFactoryArguments(repourl=repourl,
752 baseURL=baseURL,
753 defaultBranch=defaultBranch,
754 )
755 assert self.args['mode'] != "export", \
756 "Darcs does not have an 'export' mode"
757 if repourl and baseURL:
758 raise ValueError("you must provide exactly one of repourl and"
759 " baseURL")
760
761 - def startVC(self, branch, revision, patch):
804
805
807 """Check out a source tree from a git repository 'repourl'."""
808
809 name = "git"
810
811 renderables = [ 'repourl' ]
812
813 - def __init__(self, repourl=None,
814 branch="master",
815 submodules=False,
816 ignore_ignores=None,
817 reference=None,
818 shallow=False,
819 progress=False,
820 **kwargs):
821 """
822 @type repourl: string
823 @param repourl: the URL which points at the git repository
824
825 @type branch: string
826 @param branch: The branch or tag to check out by default. If
827 a build specifies a different branch, it will
828 be used instead of this.
829
830 @type submodules: boolean
831 @param submodules: Whether or not to update (and initialize)
832 git submodules.
833
834 @type reference: string
835 @param reference: The path to a reference repository to obtain
836 objects from, if any.
837
838 @type shallow: boolean
839 @param shallow: Use a shallow or clone, if possible
840
841 @type progress: boolean
842 @param progress: Pass the --progress option when fetching. This
843 can solve long fetches getting killed due to
844 lack of output, but requires Git 1.7.2+.
845 """
846 Source.__init__(self, **kwargs)
847 self.repourl = _ComputeRepositoryURL(repourl)
848 self.branch = branch
849 self.addFactoryArguments(repourl=repourl,
850 branch=branch,
851 submodules=submodules,
852 ignore_ignores=ignore_ignores,
853 reference=reference,
854 shallow=shallow,
855 progress=progress,
856 )
857 self.args.update({'submodules': submodules,
858 'ignore_ignores': ignore_ignores,
859 'reference': reference,
860 'shallow': shallow,
861 'progress': progress,
862 })
863
868
869 - def startVC(self, branch, revision, patch):
870 self.args['branch'] = branch
871 self.args['repourl'] = self.repourl
872 self.args['revision'] = revision
873 self.args['patch'] = patch
874
875
876 try:
877
878 self.args['gerrit_branch'] = self.build.getProperty("event.patchSet.ref")
879 self.setProperty("gerrit_branch", self.args['gerrit_branch'])
880 except KeyError:
881 try:
882
883 change = self.build.getProperty("gerrit_change").split('/')
884 if len(change) == 2:
885 self.args['gerrit_branch'] = "refs/changes/%2.2d/%d/%d" \
886 % (int(change[0]) % 100, int(change[0]), int(change[1]))
887 self.setProperty("gerrit_branch", self.args['gerrit_branch'])
888 except:
889 pass
890
891 slavever = self.slaveVersion("git")
892 if not slavever:
893 raise BuildSlaveTooOldError("slave is too old, does not know "
894 "about git")
895 cmd = LoggedRemoteCommand("git", self.args)
896 self.startCommand(cmd)
897
898
900 """Check out a source tree from a repo repository described by manifest."""
901
902 name = "repo"
903
904 renderables = [ "manifest_url" ]
905
906 - def __init__(self,
907 manifest_url=None,
908 manifest_branch="master",
909 manifest_file="default.xml",
910 tarball=None,
911 **kwargs):
912 """
913 @type manifest_url: string
914 @param manifest_url: The URL which points at the repo manifests repository.
915
916 @type manifest_branch: string
917 @param manifest_branch: The manifest branch to check out by default.
918
919 @type manifest_file: string
920 @param manifest_file: The manifest to use for sync.
921
922 """
923 Source.__init__(self, **kwargs)
924 self.manifest_url = _ComputeRepositoryURL(manifest_url)
925 self.addFactoryArguments(manifest_url=manifest_url,
926 manifest_branch=manifest_branch,
927 manifest_file=manifest_file,
928 tarball=tarball,
929 )
930 self.args.update({'manifest_branch': manifest_branch,
931 'manifest_file': manifest_file,
932 'tarball': tarball,
933 })
934
939
941 """
942 lets try to be nice in the format we want
943 can support several instances of "repo download proj number/patch" (direct copy paste from gerrit web site)
944 or several instances of "proj number/patch" (simpler version)
945 This feature allows integrator to build with several pending interdependant changes.
946 returns list of repo downloads sent to the buildslave
947 """
948 import re
949 if s == None:
950 return []
951 re1 = re.compile("repo download ([^ ]+) ([0-9]+/[0-9]+)")
952 re2 = re.compile("([^ ]+) ([0-9]+/[0-9]+)")
953 re3 = re.compile("([^ ]+)/([0-9]+/[0-9]+)")
954 ret = []
955 for cur_re in [re1, re2, re3]:
956 res = cur_re.search(s)
957 while res:
958 ret.append("%s %s" % (res.group(1), res.group(2)))
959 s = s[:res.start(0)] + s[res.end(0):]
960 res = cur_re.search(s)
961 return ret
962
964 """taken the changesource and forcebuild property,
965 build the repo download command to send to the slave
966 making this a defereable allow config to tweak this
967 in order to e.g. manage dependancies
968 """
969 try:
970 downloads = self.build.getProperty("repo_downloads")
971 except KeyError:
972 downloads = []
973
974
975 for change in self.build.allChanges():
976 if (change.properties.has_key("event.type") and
977 change.properties["event.type"] == "patchset-created"):
978 downloads.append("%s %s/%s"% (change.properties["event.change.project"],
979 change.properties["event.change.number"],
980 change.properties["event.patchSet.number"]))
981
982
983
984
985 for propName in ["repo_d"] + ["repo_d%d" % i for i in xrange(0,10)] + \
986 ["repo_download"] + ["repo_download%d" % i for i in xrange(0,10)]:
987 try:
988 s = self.build.getProperty(propName)
989 downloads.extend(self.parseDownloadProperty(s))
990 except KeyError:
991 pass
992
993 if downloads:
994 self.args["repo_downloads"] = downloads
995 self.setProperty("repo_downloads", downloads)
996 return defer.succeed(None)
997
998 - def startVC(self, branch, revision, patch):
1005
1013
1017
1019 if cmd.updates.has_key("repo_downloaded"):
1020 repo_downloaded = cmd.updates["repo_downloaded"][-1]
1021 if repo_downloaded:
1022 self.setProperty("repo_downloaded", str(repo_downloaded), "Source")
1023
1024
1026 """Check out a source tree from a bzr (Bazaar) repository at 'repourl'.
1027
1028 """
1029
1030 name = "bzr"
1031
1032 renderables = [ 'repourl', 'baseURL' ]
1033
1034 - def __init__(self, repourl=None, baseURL=None, defaultBranch=None,
1035 forceSharedRepo=None,
1036 **kwargs):
1037 """
1038 @type repourl: string
1039 @param repourl: the URL which points at the bzr repository. This
1040 is used as the default branch. Using C{repourl} does
1041 not enable builds of alternate branches: use
1042 C{baseURL} to enable this. Use either C{repourl} or
1043 C{baseURL}, not both.
1044
1045 @param baseURL: if branches are enabled, this is the base URL to
1046 which a branch name will be appended. It should
1047 probably end in a slash. Use exactly one of
1048 C{repourl} and C{baseURL}.
1049
1050 @param defaultBranch: if branches are enabled, this is the branch
1051 to use if the Build does not specify one
1052 explicitly. It will simply be appended to
1053 C{baseURL} and the result handed to the
1054 'bzr checkout pull' command.
1055
1056
1057 @param forceSharedRepo: Boolean, defaults to False. If set to True,
1058 the working directory will be made into a
1059 bzr shared repository if it is not already.
1060 Shared repository greatly reduces the amount
1061 of history data that needs to be downloaded
1062 if not using update/copy mode, or if using
1063 update/copy mode with multiple branches.
1064 """
1065 self.repourl = _ComputeRepositoryURL(repourl)
1066 self.baseURL = _ComputeRepositoryURL(baseURL)
1067 self.branch = defaultBranch
1068 Source.__init__(self, **kwargs)
1069 self.addFactoryArguments(repourl=repourl,
1070 baseURL=baseURL,
1071 defaultBranch=defaultBranch,
1072 forceSharedRepo=forceSharedRepo
1073 )
1074 self.args.update({'forceSharedRepo': forceSharedRepo})
1075 if repourl and baseURL:
1076 raise ValueError("you must provide exactly one of repourl and"
1077 " baseURL")
1078
1080 if not changes:
1081 return None
1082 lastChange = max([int(c.revision) for c in changes])
1083 return lastChange
1084
1085 - def startVC(self, branch, revision, patch):
1109
1110
1112 """Check out a source tree from a mercurial repository 'repourl'."""
1113
1114 name = "hg"
1115
1116 renderables = [ 'repourl', 'baseURL' ]
1117
1118 - def __init__(self, repourl=None, baseURL=None, defaultBranch=None,
1119 branchType='dirname', clobberOnBranchChange=True, **kwargs):
1120 """
1121 @type repourl: string
1122 @param repourl: the URL which points at the Mercurial repository.
1123 This uses the 'default' branch unless defaultBranch is
1124 specified below and the C{branchType} is set to
1125 'inrepo'. It is an error to specify a branch without
1126 setting the C{branchType} to 'inrepo'.
1127
1128 @param baseURL: if 'dirname' branches are enabled, this is the base URL
1129 to which a branch name will be appended. It should
1130 probably end in a slash. Use exactly one of C{repourl}
1131 and C{baseURL}.
1132
1133 @param defaultBranch: if branches are enabled, this is the branch
1134 to use if the Build does not specify one
1135 explicitly.
1136 For 'dirname' branches, It will simply be
1137 appended to C{baseURL} and the result handed to
1138 the 'hg update' command.
1139 For 'inrepo' branches, this specifies the named
1140 revision to which the tree will update after a
1141 clone.
1142
1143 @param branchType: either 'dirname' or 'inrepo' depending on whether
1144 the branch name should be appended to the C{baseURL}
1145 or the branch is a mercurial named branch and can be
1146 found within the C{repourl}
1147
1148 @param clobberOnBranchChange: boolean, defaults to True. If set and
1149 using inrepos branches, clobber the tree
1150 at each branch change. Otherwise, just
1151 update to the branch.
1152 """
1153 self.repourl = _ComputeRepositoryURL(repourl)
1154 self.baseURL = _ComputeRepositoryURL(baseURL)
1155 self.branch = defaultBranch
1156 self.branchType = branchType
1157 self.clobberOnBranchChange = clobberOnBranchChange
1158 Source.__init__(self, **kwargs)
1159 self.addFactoryArguments(repourl=repourl,
1160 baseURL=baseURL,
1161 defaultBranch=defaultBranch,
1162 branchType=branchType,
1163 clobberOnBranchChange=clobberOnBranchChange,
1164 )
1165 if repourl and baseURL:
1166 raise ValueError("you must provide exactly one of repourl and"
1167 " baseURL")
1168
1169 - def startVC(self, branch, revision, patch):
1170 slavever = self.slaveVersion("hg")
1171 if not slavever:
1172 raise BuildSlaveTooOldError("slave is too old, does not know "
1173 "about hg")
1174
1175 if self.repourl:
1176
1177 assert self.branchType == 'inrepo' or not branch
1178 self.args['repourl'] = self.repourl
1179 if branch:
1180 self.args['branch'] = branch
1181 else:
1182 self.args['repourl'] = self.baseURL + (branch or '')
1183 self.args['revision'] = revision
1184 self.args['patch'] = patch
1185 self.args['clobberOnBranchChange'] = self.clobberOnBranchChange
1186 self.args['branchType'] = self.branchType
1187
1188 revstuff = []
1189 if branch is not None and branch != self.branch:
1190 revstuff.append("[branch]")
1191 self.description.extend(revstuff)
1192 self.descriptionDone.extend(revstuff)
1193
1194 cmd = LoggedRemoteCommand("hg", self.args)
1195 self.startCommand(cmd)
1196
1198 if not changes:
1199 return None
1200
1201
1202
1203
1204 if len(changes) > 1:
1205 log.msg("Mercurial.computeSourceRevision: warning: "
1206 "there are %d changes here, assuming the last one is "
1207 "the most recent" % len(changes))
1208 return changes[-1].revision
1209
1210
1212 """ P4 is a class for accessing perforce revision control"""
1213 name = "p4"
1214
1215 renderables = [ 'p4base' ]
1216
1217 - def __init__(self, p4base=None, defaultBranch=None, p4port=None, p4user=None,
1218 p4passwd=None, p4extra_views=[], p4line_end='local',
1219 p4client='buildbot_%(slave)s_%(builder)s', **kwargs):
1220 """
1221 @type p4base: string
1222 @param p4base: A view into a perforce depot, typically
1223 "//depot/proj/"
1224
1225 @type defaultBranch: string
1226 @param defaultBranch: Identify a branch to build by default. Perforce
1227 is a view based branching system. So, the branch
1228 is normally the name after the base. For example,
1229 branch=1.0 is view=//depot/proj/1.0/...
1230 branch=1.1 is view=//depot/proj/1.1/...
1231
1232 @type p4port: string
1233 @param p4port: Specify the perforce server to connection in the format
1234 <host>:<port>. Example "perforce.example.com:1666"
1235
1236 @type p4user: string
1237 @param p4user: The perforce user to run the command as.
1238
1239 @type p4passwd: string
1240 @param p4passwd: The password for the perforce user.
1241
1242 @type p4extra_views: list of tuples
1243 @param p4extra_views: Extra views to be added to
1244 the client that is being used.
1245
1246 @type p4line_end: string
1247 @param p4line_end: value of the LineEnd client specification property
1248
1249 @type p4client: string
1250 @param p4client: The perforce client to use for this buildslave.
1251 """
1252
1253 self.p4base = _ComputeRepositoryURL(p4base)
1254 self.branch = defaultBranch
1255 Source.__init__(self, **kwargs)
1256 self.addFactoryArguments(p4base=p4base,
1257 defaultBranch=defaultBranch,
1258 p4port=p4port,
1259 p4user=p4user,
1260 p4passwd=p4passwd,
1261 p4extra_views=p4extra_views,
1262 p4line_end=p4line_end,
1263 p4client=p4client,
1264 )
1265 self.args['p4port'] = p4port
1266 self.args['p4user'] = p4user
1267 self.args['p4passwd'] = p4passwd
1268 self.args['p4extra_views'] = p4extra_views
1269 self.args['p4line_end'] = p4line_end
1270 self.p4client = p4client
1271
1278
1280 if not changes:
1281 return None
1282 lastChange = max([int(c.revision) for c in changes])
1283 return lastChange
1284
1285 - def startVC(self, branch, revision, patch):
1295
1297 """This is a partial solution for using a P4 source repository. You are
1298 required to manually set up each build slave with a useful P4
1299 environment, which means setting various per-slave environment variables,
1300 and creating a P4 client specification which maps the right files into
1301 the slave's working directory. Once you have done that, this step merely
1302 performs a 'p4 sync' to update that workspace with the newest files.
1303
1304 Each slave needs the following environment:
1305
1306 - PATH: the 'p4' binary must be on the slave's PATH
1307 - P4USER: each slave needs a distinct user account
1308 - P4CLIENT: each slave needs a distinct client specification
1309
1310 You should use 'p4 client' (?) to set up a client view spec which maps
1311 the desired files into $SLAVEBASE/$BUILDERBASE/source .
1312 """
1313
1314 name = "p4sync"
1315
1316 - def __init__(self, p4port, p4user, p4passwd, p4client, **kwargs):
1317 assert kwargs['mode'] == "copy", "P4Sync can only be used in mode=copy"
1318 self.branch = None
1319 Source.__init__(self, **kwargs)
1320 self.addFactoryArguments(p4port=p4port,
1321 p4user=p4user,
1322 p4passwd=p4passwd,
1323 p4client=p4client,
1324 )
1325 self.args['p4port'] = p4port
1326 self.args['p4user'] = p4user
1327 self.args['p4passwd'] = p4passwd
1328 self.args['p4client'] = p4client
1329
1331 if not changes:
1332 return None
1333 lastChange = max([int(c.revision) for c in changes])
1334 return lastChange
1335
1336 - def startVC(self, branch, revision, patch):
1341
1342
1344 """Check out a source tree from a monotone repository 'repourl'."""
1345
1346 name = "mtn"
1347
1348 renderables = [ 'repourl' ]
1349
1350 - def __init__(self, repourl=None, branch=None, progress=False, **kwargs):
1351 """
1352 @type repourl: string
1353 @param repourl: the URI which points at the monotone repository.
1354
1355 @type branch: string
1356 @param branch: The branch or tag to check out by default. If
1357 a build specifies a different branch, it will
1358 be used instead of this.
1359
1360 @type progress: boolean
1361 @param progress: Pass the --ticker=dot option when pulling. This
1362 can solve long fetches getting killed due to
1363 lack of output.
1364 """
1365 Source.__init__(self, **kwargs)
1366 self.repourl = _ComputeRepositoryURL(repourl)
1367 if (not repourl):
1368 raise ValueError("you must provide a repository uri in 'repourl'")
1369 if (not branch):
1370 raise ValueError("you must provide a default branch in 'branch'")
1371 self.addFactoryArguments(repourl=repourl,
1372 branch=branch,
1373 progress=progress,
1374 )
1375 self.args.update({'branch': branch,
1376 'progress': progress,
1377 })
1378
1379 - def startVC(self, branch, revision, patch):
1393
1395 if not changes:
1396 return None
1397
1398
1399
1400
1401 if len(changes) > 1:
1402 log.msg("Monotone.computeSourceRevision: warning: "
1403 "there are %d changes here, assuming the last one is "
1404 "the most recent" % len(changes))
1405 return changes[-1].revision
1406