buildbot.steps.source

1 # -*- test-case-name: buildbot.test.test_vc -*- 2 3 from warnings import warn 4 from email.Utils import formatdate 5 from twisted.python import log 6 from buildbot.process.buildstep import LoggingBuildStep, LoggedRemoteCommand 7 from buildbot.interfaces import BuildSlaveTooOldError 8 from buildbot.status.builder import SKIPPED 9 10

11 -class Source(LoggingBuildStep):

12 """This is a base class to generate a source tree in the buildslave. 13 Each version control system has a specialized subclass, and is expected 14 to override __init__ and implement computeSourceRevision() and 15 startVC(). The class as a whole builds up the self.args dictionary, then 16 starts a LoggedRemoteCommand with those arguments. 17 """ 18 19 # if the checkout fails, there's no point in doing anything else 20 haltOnFailure = True 21 flunkOnFailure = True 22 notReally = False 23 24 branch = None # the default branch, should be set in __init__ 25

26 - def __init__(self, workdir=None, mode='update', alwaysUseLatest=False, 27 timeout=20*60, retry=None, **kwargs):

28 """ 29 @type workdir: string 30 @param workdir: local directory (relative to the Builder's root) 31 where the tree should be placed 32 33 @type mode: string 34 @param mode: the kind of VC operation that is desired: 35 - 'update': specifies that the checkout/update should be 36 performed directly into the workdir. Each build is performed 37 in the same directory, allowing for incremental builds. This 38 minimizes disk space, bandwidth, and CPU time. However, it 39 may encounter problems if the build process does not handle 40 dependencies properly (if you must sometimes do a 'clean 41 build' to make sure everything gets compiled), or if source 42 files are deleted but generated files can influence test 43 behavior (e.g. python's .pyc files), or when source 44 directories are deleted but generated files prevent CVS from 45 removing them. When used with a patched checkout, from a 46 previous buildbot try for instance, it will try to "revert" 47 the changes first and will do a clobber if it is unable to 48 get a clean checkout. The behavior is SCM-dependent. 49 50 - 'copy': specifies that the source-controlled workspace 51 should be maintained in a separate directory (called the 52 'copydir'), using checkout or update as necessary. For each 53 build, a new workdir is created with a copy of the source 54 tree (rm -rf workdir; cp -R -P -p copydir workdir). This 55 doubles the disk space required, but keeps the bandwidth low 56 (update instead of a full checkout). A full 'clean' build 57 is performed each time. This avoids any generated-file 58 build problems, but is still occasionally vulnerable to 59 problems such as a CVS repository being manually rearranged 60 (causing CVS errors on update) which are not an issue with 61 a full checkout. 62 63 - 'clobber': specifies that the working directory should be 64 deleted each time, necessitating a full checkout for each 65 build. This insures a clean build off a complete checkout, 66 avoiding any of the problems described above, but is 67 bandwidth intensive, as the whole source tree must be 68 pulled down for each build. 69 70 - 'export': is like 'clobber', except that e.g. the 'cvs 71 export' command is used to create the working directory. 72 This command removes all VC metadata files (the 73 CVS/.svn/{arch} directories) from the tree, which is 74 sometimes useful for creating source tarballs (to avoid 75 including the metadata in the tar file). Not all VC systems 76 support export. 77 78 @type alwaysUseLatest: boolean 79 @param alwaysUseLatest: whether to always update to the most 80 recent available sources for this build. 81 82 Normally the Source step asks its Build for a list of all 83 Changes that are supposed to go into the build, then computes a 84 'source stamp' (revision number or timestamp) that will cause 85 exactly that set of changes to be present in the checked out 86 tree. This is turned into, e.g., 'cvs update -D timestamp', or 87 'svn update -r revnum'. If alwaysUseLatest=True, bypass this 88 computation and always update to the latest available sources 89 for each build. 90 91 The source stamp helps avoid a race condition in which someone 92 commits a change after the master has decided to start a build 93 but before the slave finishes checking out the sources. At best 94 this results in a build which contains more changes than the 95 buildmaster thinks it has (possibly resulting in the wrong 96 person taking the blame for any problems that result), at worst 97 is can result in an incoherent set of sources (splitting a 98 non-atomic commit) which may not build at all. 99 100 @type retry: tuple of ints (delay, repeats) (or None) 101 @param retry: if provided, VC update failures are re-attempted up 102 to REPEATS times, with DELAY seconds between each 103 attempt. Some users have slaves with poor connectivity 104 to their VC repository, and they say that up to 80% of 105 their build failures are due to transient network 106 failures that could be handled by simply retrying a 107 couple times. 108 109 """ 110 111 LoggingBuildStep.__init__(self, **kwargs) 112 self.addFactoryArguments(workdir=workdir, 113 mode=mode, 114 alwaysUseLatest=alwaysUseLatest, 115 timeout=timeout, 116 retry=retry, 117 ) 118 119 assert mode in ("update", "copy", "clobber", "export") 120 if retry: 121 delay, repeats = retry 122 assert isinstance(repeats, int) 123 assert repeats > 0 124 self.args = {'mode': mode, 125 'workdir': workdir, 126 'timeout': timeout, 127 'retry': retry, 128 'patch': None, # set during .start 129 } 130 self.alwaysUseLatest = alwaysUseLatest 131 132 # Compute defaults for descriptions: 133 description = ["updating"] 134 descriptionDone = ["update"] 135 if mode == "clobber": 136 description = ["checkout"] 137 # because checkingouting takes too much space 138 descriptionDone = ["checkout"] 139 elif mode == "export": 140 description = ["exporting"] 141 descriptionDone = ["export"] 142 self.description = description 143 self.descriptionDone = descriptionDone

144

145 - def setDefaultWorkdir(self, workdir):

146 self.args['workdir'] = self.args['workdir'] or workdir

147

148 - def describe(self, done=False):

149 if done: 150 return self.descriptionDone 151 return self.description

152

153 - def computeSourceRevision(self, changes):

154 """Each subclass must implement this method to do something more 155 precise than -rHEAD every time. For version control systems that use 156 repository-wide change numbers (SVN, P4), this can simply take the 157 maximum such number from all the changes involved in this build. For 158 systems that do not (CVS), it needs to create a timestamp based upon 159 the latest Change, the Build's treeStableTimer, and an optional 160 self.checkoutDelay value.""" 161 return None

162

163 - def start(self):

164 if self.notReally: 165 log.msg("faking %s checkout/update" % self.name) 166 self.step_status.setText(["fake", self.name, "successful"]) 167 self.addCompleteLog("log", 168 "Faked %s checkout/update 'successful'\n" \ 169 % self.name) 170 return SKIPPED 171 172 # what source stamp would this build like to use? 173 s = self.build.getSourceStamp() 174 # if branch is None, then use the Step's "default" branch 175 branch = s.branch or self.branch 176 # if revision is None, use the latest sources (-rHEAD) 177 revision = s.revision 178 if not revision and not self.alwaysUseLatest: 179 revision = self.computeSourceRevision(s.changes) 180 # if patch is None, then do not patch the tree after checkout 181 182 # 'patch' is None or a tuple of (patchlevel, diff, root) 183 # root is optional. 184 patch = s.patch 185 if patch: 186 self.addCompleteLog("patch", patch[1]) 187 188 if self.alwaysUseLatest: 189 revision = None 190 self.startVC(branch, revision, patch)

191

192 - def commandComplete(self, cmd):

193 if cmd.updates.has_key("got_revision"): 194 got_revision = cmd.updates["got_revision"][-1] 195 if got_revision is not None: 196 self.setProperty("got_revision", str(got_revision), "Source")

197 198 199

200 -class CVS(Source):

201 """I do CVS checkout/update operations. 202 203 Note: if you are doing anonymous/pserver CVS operations, you will need 204 to manually do a 'cvs login' on each buildslave before the slave has any 205 hope of success. XXX: fix then, take a cvs password as an argument and 206 figure out how to do a 'cvs login' on each build 207 """ 208 209 name = "cvs" 210 211 #progressMetrics = ('output',) 212 # 213 # additional things to track: update gives one stderr line per directory 214 # (starting with 'cvs server: Updating ') (and is fairly stable if files 215 # is empty), export gives one line per directory (starting with 'cvs 216 # export: Updating ') and another line per file (starting with U). Would 217 # be nice to track these, requires grepping LogFile data for lines, 218 # parsing each line. Might be handy to have a hook in LogFile that gets 219 # called with each complete line. 220

221 - def __init__(self, cvsroot, cvsmodule, 222 global_options=[], branch=None, checkoutDelay=None, 223 checkout_options=[], 224 login=None, 225 **kwargs):

226 227 """ 228 @type cvsroot: string 229 @param cvsroot: CVS Repository from which the source tree should 230 be obtained. '/home/warner/Repository' for local 231 or NFS-reachable repositories, 232 ':pserver:anon@foo.com:/cvs' for anonymous CVS, 233 'user@host.com:/cvs' for non-anonymous CVS or 234 CVS over ssh. Lots of possibilities, check the 235 CVS documentation for more. 236 237 @type cvsmodule: string 238 @param cvsmodule: subdirectory of CVS repository that should be 239 retrieved 240 241 @type login: string or None 242 @param login: if not None, a string which will be provided as a 243 password to the 'cvs login' command, used when a 244 :pserver: method is used to access the repository. 245 This login is only needed once, but must be run 246 each time (just before the CVS operation) because 247 there is no way for the buildslave to tell whether 248 it was previously performed or not. 249 250 @type branch: string 251 @param branch: the default branch name, will be used in a '-r' 252 argument to specify which branch of the source tree 253 should be used for this checkout. Defaults to None, 254 which means to use 'HEAD'. 255 256 @type checkoutDelay: int or None 257 @param checkoutDelay: if not None, the number of seconds to put 258 between the last known Change and the 259 timestamp given to the -D argument. This 260 defaults to exactly half of the parent 261 Build's .treeStableTimer, but it could be 262 set to something else if your CVS change 263 notification has particularly weird 264 latency characteristics. 265 266 @type global_options: list of strings 267 @param global_options: these arguments are inserted in the cvs 268 command line, before the 269 'checkout'/'update' command word. See 270 'cvs --help-options' for a list of what 271 may be accepted here. ['-r'] will make 272 the checked out files read only. ['-r', 273 '-R'] will also assume the repository is 274 read-only (I assume this means it won't 275 use locks to insure atomic access to the 276 ,v files). 277 278 @type checkout_options: list of strings 279 @param checkout_options: these arguments are inserted in the cvs 280 command line, after 'checkout' but before 281 branch or revision specifiers. 282 """ 283 284 self.checkoutDelay = checkoutDelay 285 self.branch = branch 286 287 Source.__init__(self, **kwargs) 288 self.addFactoryArguments(cvsroot=cvsroot, 289 cvsmodule=cvsmodule, 290 global_options=global_options, 291 checkout_options=checkout_options, 292 branch=branch, 293 checkoutDelay=checkoutDelay, 294 login=login, 295 ) 296 297 self.args.update({'cvsroot': cvsroot, 298 'cvsmodule': cvsmodule, 299 'global_options': global_options, 300 'checkout_options':checkout_options, 301 'login': login, 302 })

303

304 - def computeSourceRevision(self, changes):

305 if not changes: 306 return None 307 lastChange = max([c.when for c in changes]) 308 if self.checkoutDelay is not None: 309 when = lastChange + self.checkoutDelay 310 else: 311 lastSubmit = max([r.submittedAt for r in self.build.requests]) 312 when = (lastChange + lastSubmit) / 2 313 return formatdate(when)

314

315 - def startVC(self, branch, revision, patch):

316 if self.slaveVersionIsOlderThan("cvs", "1.39"): 317 # the slave doesn't know to avoid re-using the same sourcedir 318 # when the branch changes. We have no way of knowing which branch 319 # the last build used, so if we're using a non-default branch and 320 # either 'update' or 'copy' modes, it is safer to refuse to 321 # build, and tell the user they need to upgrade the buildslave. 322 if (branch != self.branch 323 and self.args['mode'] in ("update", "copy")): 324 m = ("This buildslave (%s) does not know about multiple " 325 "branches, and using mode=%s would probably build the " 326 "wrong tree. " 327 "Refusing to build. Please upgrade the buildslave to " 328 "buildbot-0.7.0 or newer." % (self.build.slavename, 329 self.args['mode'])) 330 log.msg(m) 331 raise BuildSlaveTooOldError(m) 332 333 if branch is None: 334 branch = "HEAD" 335 self.args['branch'] = branch 336 self.args['revision'] = revision 337 self.args['patch'] = patch 338 339 if self.args['branch'] == "HEAD" and self.args['revision']: 340 # special case. 'cvs update -r HEAD -D today' gives no files 341 # TODO: figure out why, see if it applies to -r BRANCH 342 self.args['branch'] = None 343 344 # deal with old slaves 345 warnings = [] 346 slavever = self.slaveVersion("cvs", "old") 347 348 if slavever == "old": 349 # 0.5.0 350 if self.args['mode'] == "export": 351 self.args['export'] = 1 352 elif self.args['mode'] == "clobber": 353 self.args['clobber'] = 1 354 elif self.args['mode'] == "copy": 355 self.args['copydir'] = "source" 356 self.args['tag'] = self.args['branch'] 357 assert not self.args['patch'] # 0.5.0 slave can't do patch 358 359 cmd = LoggedRemoteCommand("cvs", self.args) 360 self.startCommand(cmd, warnings)

361 362

363 -class SVN(Source):

364 """I perform Subversion checkout/update operations.""" 365 366 name = 'svn' 367

368 - def __init__(self, svnurl=None, baseURL=None, defaultBranch=None, 369 directory=None, username=None, password=None, 370 extra_args=None, keep_on_purge=None, ignore_ignores=None, 371 always_purge=None, depth=None, **kwargs):

372 """ 373 @type svnurl: string 374 @param svnurl: the URL which points to the Subversion server, 375 combining the access method (HTTP, ssh, local file), 376 the repository host/port, the repository path, the 377 sub-tree within the repository, and the branch to 378 check out. Using C{svnurl} does not enable builds of 379 alternate branches: use C{baseURL} to enable this. 380 Use exactly one of C{svnurl} and C{baseURL}. 381 382 @param baseURL: if branches are enabled, this is the base URL to 383 which a branch name will be appended. It should 384 probably end in a slash. Use exactly one of 385 C{svnurl} and C{baseURL}. 386 387 @param defaultBranch: if branches are enabled, this is the branch 388 to use if the Build does not specify one 389 explicitly. It will simply be appended 390 to C{baseURL} and the result handed to 391 the SVN command. 392 393 @param username: username to pass to svn's --username 394 @param password: username to pass to svn's --password 395 """ 396 397 if not kwargs.has_key('workdir') and directory is not None: 398 # deal with old configs 399 warn("Please use workdir=, not directory=", DeprecationWarning) 400 kwargs['workdir'] = directory 401 402 self.svnurl = svnurl 403 self.baseURL = baseURL 404 self.branch = defaultBranch 405 self.username = username 406 self.password = password 407 self.extra_args = extra_args 408 self.keep_on_purge = keep_on_purge 409 self.ignore_ignores = ignore_ignores 410 self.always_purge = always_purge 411 self.depth = depth 412 413 Source.__init__(self, **kwargs) 414 self.addFactoryArguments(svnurl=svnurl, 415 baseURL=baseURL, 416 defaultBranch=defaultBranch, 417 directory=directory, 418 username=username, 419 password=password, 420 extra_args=extra_args, 421 keep_on_purge=keep_on_purge, 422 ignore_ignores=ignore_ignores, 423 always_purge=always_purge, 424 depth=depth, 425 ) 426 427 if not svnurl and not baseURL: 428 raise ValueError("you must use exactly one of svnurl and baseURL")

429 430

431 - def computeSourceRevision(self, changes):

432 if not changes or None in [c.revision for c in changes]: 433 return None 434 lastChange = max([int(c.revision) for c in changes]) 435 return lastChange

436

437 - def startVC(self, branch, revision, patch):

438 439 # handle old slaves 440 warnings = [] 441 slavever = self.slaveVersion("svn", "old") 442 if not slavever: 443 m = "slave does not have the 'svn' command" 444 raise BuildSlaveTooOldError(m) 445 446 if self.slaveVersionIsOlderThan("svn", "1.39"): 447 # the slave doesn't know to avoid re-using the same sourcedir 448 # when the branch changes. We have no way of knowing which branch 449 # the last build used, so if we're using a non-default branch and 450 # either 'update' or 'copy' modes, it is safer to refuse to 451 # build, and tell the user they need to upgrade the buildslave. 452 if (branch != self.branch 453 and self.args['mode'] in ("update", "copy")): 454 m = ("This buildslave (%s) does not know about multiple " 455 "branches, and using mode=%s would probably build the " 456 "wrong tree. " 457 "Refusing to build. Please upgrade the buildslave to " 458 "buildbot-0.7.0 or newer." % (self.build.slavename, 459 self.args['mode'])) 460 raise BuildSlaveTooOldError(m) 461 462 if slavever == "old": 463 # 0.5.0 compatibility 464 if self.args['mode'] in ("clobber", "copy"): 465 # TODO: use some shell commands to make up for the 466 # deficiency, by blowing away the old directory first (thus 467 # forcing a full checkout) 468 warnings.append("WARNING: this slave can only do SVN updates" 469 ", not mode=%s\n" % self.args['mode']) 470 log.msg("WARNING: this slave only does mode=update") 471 if self.args['mode'] == "export": 472 raise BuildSlaveTooOldError("old slave does not have " 473 "mode=export") 474 self.args['directory'] = self.args['workdir'] 475 if revision is not None: 476 # 0.5.0 can only do HEAD. We have no way of knowing whether 477 # the requested revision is HEAD or not, and for 478 # slowly-changing trees this will probably do the right 479 # thing, so let it pass with a warning 480 m = ("WARNING: old slave can only update to HEAD, not " 481 "revision=%s" % revision) 482 log.msg(m) 483 warnings.append(m + "\n") 484 revision = "HEAD" # interprets this key differently 485 if patch: 486 raise BuildSlaveTooOldError("old slave can't do patch") 487 488 if self.svnurl: 489 assert not branch # we need baseURL= to use branches 490 self.args['svnurl'] = self.svnurl 491 else: 492 self.args['svnurl'] = self.baseURL + branch 493 self.args['revision'] = revision 494 self.args['patch'] = patch 495 496 #Set up depth if specified 497 if self.depth is not None: 498 if self.slaveVersionIsOlderThan("svn","2.9"): 499 m = ("This buildslave (%s) does not support svn depth " 500 "arguments. " 501 "Refusing to build. " 502 "Please upgrade the buildslave." % (self.build.slavename)) 503 raise BuildSlaveTooOldError(m) 504 else: 505 self.args['depth'] = self.depth 506 507 if self.username is not None or self.password is not None: 508 if self.slaveVersionIsOlderThan("svn", "2.8"): 509 m = ("This buildslave (%s) does not support svn usernames " 510 "and passwords. " 511 "Refusing to build. Please upgrade the buildslave to " 512 "buildbot-0.7.10 or newer." % (self.build.slavename,)) 513 raise BuildSlaveTooOldError(m) 514 if self.username is not None: self.args['username'] = self.username 515 if self.password is not None: self.args['password'] = self.password 516 517 if self.extra_args is not None: 518 self.args['extra_args'] = self.extra_args 519 520 revstuff = [] 521 if branch is not None and branch != self.branch: 522 revstuff.append("[branch]") 523 if revision is not None: 524 revstuff.append("r%s" % revision) 525 if patch is not None: 526 revstuff.append("[patch]") 527 self.description.extend(revstuff) 528 self.descriptionDone.extend(revstuff) 529 530 cmd = LoggedRemoteCommand("svn", self.args) 531 self.startCommand(cmd, warnings)

532 533

534 -class Darcs(Source):

535 """Check out a source tree from a Darcs repository at 'repourl'. 536 537 Darcs has no concept of file modes. This means the eXecute-bit will be 538 cleared on all source files. As a result, you may need to invoke 539 configuration scripts with something like: 540 541 C{s(step.Configure, command=['/bin/sh', './configure'])} 542 """ 543 544 name = "darcs" 545

546 - def __init__(self, repourl=None, baseURL=None, defaultBranch=None, 547 **kwargs):

548 """ 549 @type repourl: string 550 @param repourl: the URL which points at the Darcs repository. This 551 is used as the default branch. Using C{repourl} does 552 not enable builds of alternate branches: use 553 C{baseURL} to enable this. Use either C{repourl} or 554 C{baseURL}, not both. 555 556 @param baseURL: if branches are enabled, this is the base URL to 557 which a branch name will be appended. It should 558 probably end in a slash. Use exactly one of 559 C{repourl} and C{baseURL}. 560 561 @param defaultBranch: if branches are enabled, this is the branch 562 to use if the Build does not specify one 563 explicitly. It will simply be appended to 564 C{baseURL} and the result handed to the 565 'darcs pull' command. 566 """ 567 self.repourl = repourl 568 self.baseURL = baseURL 569 self.branch = defaultBranch 570 Source.__init__(self, **kwargs) 571 self.addFactoryArguments(repourl=repourl, 572 baseURL=baseURL, 573 defaultBranch=defaultBranch, 574 ) 575 assert self.args['mode'] != "export", \ 576 "Darcs does not have an 'export' mode" 577 if (not repourl and not baseURL) or (repourl and baseURL): 578 raise ValueError("you must provide exactly one of repourl and" 579 " baseURL")

580

581 - def startVC(self, branch, revision, patch):

582 slavever = self.slaveVersion("darcs") 583 if not slavever: 584 m = "slave is too old, does not know about darcs" 585 raise BuildSlaveTooOldError(m) 586 587 if self.slaveVersionIsOlderThan("darcs", "1.39"): 588 if revision: 589 # TODO: revisit this once we implement computeSourceRevision 590 m = "0.6.6 slaves can't handle args['revision']" 591 raise BuildSlaveTooOldError(m) 592 593 # the slave doesn't know to avoid re-using the same sourcedir 594 # when the branch changes. We have no way of knowing which branch 595 # the last build used, so if we're using a non-default branch and 596 # either 'update' or 'copy' modes, it is safer to refuse to 597 # build, and tell the user they need to upgrade the buildslave. 598 if (branch != self.branch 599 and self.args['mode'] in ("update", "copy")): 600 m = ("This buildslave (%s) does not know about multiple " 601 "branches, and using mode=%s would probably build the " 602 "wrong tree. " 603 "Refusing to build. Please upgrade the buildslave to " 604 "buildbot-0.7.0 or newer." % (self.build.slavename, 605 self.args['mode'])) 606 raise BuildSlaveTooOldError(m) 607 608 if self.repourl: 609 assert not branch # we need baseURL= to use branches 610 self.args['repourl'] = self.repourl 611 else: 612 self.args['repourl'] = self.baseURL + branch 613 self.args['revision'] = revision 614 self.args['patch'] = patch 615 616 revstuff = [] 617 if branch is not None and branch != self.branch: 618 revstuff.append("[branch]") 619 self.description.extend(revstuff) 620 self.descriptionDone.extend(revstuff) 621 622 cmd = LoggedRemoteCommand("darcs", self.args) 623 self.startCommand(cmd)

624 625

626 -class Git(Source):

627 """Check out a source tree from a git repository 'repourl'.""" 628 629 name = "git" 630

631 - def __init__(self, repourl, 632 branch="master", 633 submodules=False, 634 ignore_ignores=None, 635 **kwargs):

636 """ 637 @type repourl: string 638 @param repourl: the URL which points at the git repository 639 640 @type branch: string 641 @param branch: The branch or tag to check out by default. If 642 a build specifies a different branch, it will 643 be used instead of this. 644 645 @type submodules: boolean 646 @param submodules: Whether or not to update (and initialize) 647 git submodules. 648 649 """ 650 Source.__init__(self, **kwargs) 651 self.addFactoryArguments(repourl=repourl, 652 branch=branch, 653 submodules=submodules, 654 ignore_ignores=ignore_ignores, 655 ) 656 self.args.update({'repourl': repourl, 657 'branch': branch, 658 'submodules': submodules, 659 'ignore_ignores': ignore_ignores, 660 })

661

662 - def computeSourceRevision(self, changes):

663 if not changes: 664 return None 665 return changes[-1].revision

666

667 - def startVC(self, branch, revision, patch):

668 if branch is not None: 669 self.args['branch'] = branch 670 671 self.args['revision'] = revision 672 self.args['patch'] = patch 673 slavever = self.slaveVersion("git") 674 if not slavever: 675 raise BuildSlaveTooOldError("slave is too old, does not know " 676 "about git") 677 cmd = LoggedRemoteCommand("git", self.args) 678 self.startCommand(cmd)

679 680

681 -class Arch(Source):

682 """Check out a source tree from an Arch repository named 'archive' 683 available at 'url'. 'version' specifies which version number (development 684 line) will be used for the checkout: this is mostly equivalent to a 685 branch name. This version uses the 'tla' tool to do the checkout, to use 686 'baz' see L{Bazaar} instead. 687 """ 688 689 name = "arch" 690 # TODO: slaves >0.6.6 will accept args['build-config'], so use it 691

692 - def __init__(self, url, version, archive=None, **kwargs):

693 """ 694 @type url: string 695 @param url: the Arch coordinates of the repository. This is 696 typically an http:// URL, but could also be the absolute 697 pathname of a local directory instead. 698 699 @type version: string 700 @param version: the category--branch--version to check out. This is 701 the default branch. If a build specifies a different 702 branch, it will be used instead of this. 703 704 @type archive: string 705 @param archive: The archive name. If provided, it must match the one 706 that comes from the repository. If not, the 707 repository's default will be used. 708 """ 709 self.branch = version 710 Source.__init__(self, **kwargs) 711 self.addFactoryArguments(url=url, 712 version=version, 713 archive=archive, 714 ) 715 self.args.update({'url': url, 716 'archive': archive, 717 })

718

719 - def computeSourceRevision(self, changes):

720 # in Arch, fully-qualified revision numbers look like: 721 # arch@buildbot.sourceforge.net--2004/buildbot--dev--0--patch-104 722 # For any given builder, all of this is fixed except the patch-104. 723 # The Change might have any part of the fully-qualified string, so we 724 # just look for the last part. We return the "patch-NN" string. 725 if not changes: 726 return None 727 lastChange = None 728 for c in changes: 729 if not c.revision: 730 continue 731 if c.revision.endswith("--base-0"): 732 rev = 0 733 else: 734 i = c.revision.rindex("patch") 735 rev = int(c.revision[i+len("patch-"):]) 736 lastChange = max(lastChange, rev) 737 if lastChange is None: 738 return None 739 if lastChange == 0: 740 return "base-0" 741 return "patch-%d" % lastChange

742

743 - def checkSlaveVersion(self, cmd, branch):

744 warnings = [] 745 slavever = self.slaveVersion(cmd) 746 if not slavever: 747 m = "slave is too old, does not know about %s" % cmd 748 raise BuildSlaveTooOldError(m) 749 750 # slave 1.28 and later understand 'revision' 751 if self.slaveVersionIsOlderThan(cmd, "1.28"): 752 if not self.alwaysUseLatest: 753 # we don't know whether our requested revision is the latest 754 # or not. If the tree does not change very quickly, this will 755 # probably build the right thing, so emit a warning rather 756 # than refuse to build at all 757 m = "WARNING, buildslave is too old to use a revision" 758 log.msg(m) 759 warnings.append(m + "\n") 760 761 if self.slaveVersionIsOlderThan(cmd, "1.39"): 762 # the slave doesn't know to avoid re-using the same sourcedir 763 # when the branch changes. We have no way of knowing which branch 764 # the last build used, so if we're using a non-default branch and 765 # either 'update' or 'copy' modes, it is safer to refuse to 766 # build, and tell the user they need to upgrade the buildslave. 767 if (branch != self.branch 768 and self.args['mode'] in ("update", "copy")): 769 m = ("This buildslave (%s) does not know about multiple " 770 "branches, and using mode=%s would probably build the " 771 "wrong tree. " 772 "Refusing to build. Please upgrade the buildslave to " 773 "buildbot-0.7.0 or newer." % (self.build.slavename, 774 self.args['mode'])) 775 log.msg(m) 776 raise BuildSlaveTooOldError(m) 777 778 return warnings

779

780 - def startVC(self, branch, revision, patch):

781 self.args['version'] = branch 782 self.args['revision'] = revision 783 self.args['patch'] = patch 784 warnings = self.checkSlaveVersion("arch", branch) 785 786 revstuff = [] 787 if branch is not None and branch != self.branch: 788 revstuff.append("[branch]") 789 if revision is not None: 790 revstuff.append("patch%s" % revision) 791 self.description.extend(revstuff) 792 self.descriptionDone.extend(revstuff) 793 794 cmd = LoggedRemoteCommand("arch", self.args) 795 self.startCommand(cmd, warnings)

796 797

798 -class Bazaar(Arch):

799 """Bazaar is an alternative client for Arch repositories. baz is mostly 800 compatible with tla, but archive registration is slightly different.""" 801 802 # TODO: slaves >0.6.6 will accept args['build-config'], so use it 803

804 - def __init__(self, url, version, archive, **kwargs):

805 """ 806 @type url: string 807 @param url: the Arch coordinates of the repository. This is 808 typically an http:// URL, but could also be the absolute 809 pathname of a local directory instead. 810 811 @type version: string 812 @param version: the category--branch--version to check out 813 814 @type archive: string 815 @param archive: The archive name (required). This must always match 816 the one that comes from the repository, otherwise the 817 buildslave will attempt to get sources from the wrong 818 archive. 819 """ 820 self.branch = version 821 Source.__init__(self, **kwargs) 822 self.addFactoryArguments(url=url, 823 version=version, 824 archive=archive, 825 ) 826 self.args.update({'url': url, 827 'archive': archive, 828 })

829

830 - def startVC(self, branch, revision, patch):

831 self.args['version'] = branch 832 self.args['revision'] = revision 833 self.args['patch'] = patch 834 warnings = self.checkSlaveVersion("bazaar", branch) 835 836 revstuff = [] 837 if branch is not None and branch != self.branch: 838 revstuff.append("[branch]") 839 if revision is not None: 840 revstuff.append("patch%s" % revision) 841 self.description.extend(revstuff) 842 self.descriptionDone.extend(revstuff) 843 844 cmd = LoggedRemoteCommand("bazaar", self.args) 845 self.startCommand(cmd, warnings)

846

847 -class Bzr(Source):

848 """Check out a source tree from a bzr (Bazaar) repository at 'repourl'. 849 850 """ 851 852 name = "bzr" 853

854 - def __init__(self, repourl=None, baseURL=None, defaultBranch=None, 855 forceSharedRepo=None, 856 **kwargs):

857 """ 858 @type repourl: string 859 @param repourl: the URL which points at the bzr repository. This 860 is used as the default branch. Using C{repourl} does 861 not enable builds of alternate branches: use 862 C{baseURL} to enable this. Use either C{repourl} or 863 C{baseURL}, not both. 864 865 @param baseURL: if branches are enabled, this is the base URL to 866 which a branch name will be appended. It should 867 probably end in a slash. Use exactly one of 868 C{repourl} and C{baseURL}. 869 870 @param defaultBranch: if branches are enabled, this is the branch 871 to use if the Build does not specify one 872 explicitly. It will simply be appended to 873 C{baseURL} and the result handed to the 874 'bzr checkout pull' command. 875 876 877 @param forceSharedRepo: Boolean, defaults to False. If set to True, 878 the working directory will be made into a 879 bzr shared repository if it is not already. 880 Shared repository greatly reduces the amount 881 of history data that needs to be downloaded 882 if not using update/copy mode, or if using 883 update/copy mode with multiple branches. 884 """ 885 self.repourl = repourl 886 self.baseURL = baseURL 887 self.branch = defaultBranch 888 Source.__init__(self, **kwargs) 889 self.addFactoryArguments(repourl=repourl, 890 baseURL=baseURL, 891 defaultBranch=defaultBranch, 892 forceSharedRepo=forceSharedRepo 893 ) 894 self.args.update({'forceSharedRepo': forceSharedRepo}) 895 if (not repourl and not baseURL) or (repourl and baseURL): 896 raise ValueError("you must provide exactly one of repourl and" 897 " baseURL")

898

899 - def computeSourceRevision(self, changes):

900 if not changes: 901 return None 902 lastChange = max([int(c.revision) for c in changes]) 903 return lastChange

904

905 - def startVC(self, branch, revision, patch):

906 slavever = self.slaveVersion("bzr") 907 if not slavever: 908 m = "slave is too old, does not know about bzr" 909 raise BuildSlaveTooOldError(m) 910 911 if self.repourl: 912 assert not branch # we need baseURL= to use branches 913 self.args['repourl'] = self.repourl 914 else: 915 self.args['repourl'] = self.baseURL + branch 916 self.args['revision'] = revision 917 self.args['patch'] = patch 918 919 revstuff = [] 920 if branch is not None and branch != self.branch: 921 revstuff.append("[" + branch + "]") 922 if revision is not None: 923 revstuff.append("r%s" % revision) 924 self.description.extend(revstuff) 925 self.descriptionDone.extend(revstuff) 926 927 cmd = LoggedRemoteCommand("bzr", self.args) 928 self.startCommand(cmd)

929 930

931 -class Mercurial(Source):

932 """Check out a source tree from a mercurial repository 'repourl'.""" 933 934 name = "hg" 935

936 - def __init__(self, repourl=None, baseURL=None, defaultBranch=None, 937 branchType='dirname', clobberOnBranchChange=True, **kwargs):

938 """ 939 @type repourl: string 940 @param repourl: the URL which points at the Mercurial repository. 941 This uses the 'default' branch unless defaultBranch is 942 specified below and the C{branchType} is set to 943 'inrepo'. It is an error to specify a branch without 944 setting the C{branchType} to 'inrepo'. 945 946 @param baseURL: if 'dirname' branches are enabled, this is the base URL 947 to which a branch name will be appended. It should 948 probably end in a slash. Use exactly one of C{repourl} 949 and C{baseURL}. 950 951 @param defaultBranch: if branches are enabled, this is the branch 952 to use if the Build does not specify one 953 explicitly. 954 For 'dirname' branches, It will simply be 955 appended to C{baseURL} and the result handed to 956 the 'hg update' command. 957 For 'inrepo' branches, this specifies the named 958 revision to which the tree will update after a 959 clone. 960 961 @param branchType: either 'dirname' or 'inrepo' depending on whether 962 the branch name should be appended to the C{baseURL} 963 or the branch is a mercurial named branch and can be 964 found within the C{repourl} 965 966 @param clobberOnBranchChange: boolean, defaults to True. If set and 967 using inrepos branches, clobber the tree 968 at each branch change. Otherwise, just 969 update to the branch. 970 """ 971 self.repourl = repourl 972 self.baseURL = baseURL 973 self.branch = defaultBranch 974 self.branchType = branchType 975 self.clobberOnBranchChange = clobberOnBranchChange 976 Source.__init__(self, **kwargs) 977 self.addFactoryArguments(repourl=repourl, 978 baseURL=baseURL, 979 defaultBranch=defaultBranch, 980 branchType=branchType, 981 clobberOnBranchChange=clobberOnBranchChange, 982 ) 983 if (not repourl and not baseURL) or (repourl and baseURL): 984 raise ValueError("you must provide exactly one of repourl and" 985 " baseURL")

986

987 - def startVC(self, branch, revision, patch):

988 slavever = self.slaveVersion("hg") 989 if not slavever: 990 raise BuildSlaveTooOldError("slave is too old, does not know " 991 "about hg") 992 993 if self.repourl: 994 # we need baseURL= to use dirname branches 995 assert self.branchType == 'inrepo' or not branch 996 self.args['repourl'] = self.repourl 997 if branch: 998 self.args['branch'] = branch 999 else: 1000 self.args['repourl'] = self.baseURL + (branch or '') 1001 self.args['revision'] = revision 1002 self.args['patch'] = patch 1003 self.args['clobberOnBranchChange'] = self.clobberOnBranchChange 1004 self.args['branchType'] = self.branchType 1005 1006 revstuff = [] 1007 if branch is not None and branch != self.branch: 1008 revstuff.append("[branch]") 1009 self.description.extend(revstuff) 1010 self.descriptionDone.extend(revstuff) 1011 1012 cmd = LoggedRemoteCommand("hg", self.args) 1013 self.startCommand(cmd)

1014

1015 - def computeSourceRevision(self, changes):

1016 if not changes: 1017 return None 1018 # without knowing the revision ancestry graph, we can't sort the 1019 # changes at all. So for now, assume they were given to us in sorted 1020 # order, and just pay attention to the last one. See ticket #103 for 1021 # more details. 1022 if len(changes) > 1: 1023 log.msg("Mercurial.computeSourceRevision: warning: " 1024 "there are %d changes here, assuming the last one is " 1025 "the most recent" % len(changes)) 1026 return changes[-1].revision

1027 1028

1029 -class P4(Source):

1030 """ P4 is a class for accessing perforce revision control""" 1031 name = "p4" 1032

1033 - def __init__(self, p4base, defaultBranch=None, p4port=None, p4user=None, 1034 p4passwd=None, p4extra_views=[], 1035 p4client='buildbot_%(slave)s_%(builder)s', **kwargs):

1036 """ 1037 @type p4base: string 1038 @param p4base: A view into a perforce depot, typically 1039 "//depot/proj/" 1040 1041 @type defaultBranch: string 1042 @param defaultBranch: Identify a branch to build by default. Perforce 1043 is a view based branching system. So, the branch 1044 is normally the name after the base. For example, 1045 branch=1.0 is view=//depot/proj/1.0/... 1046 branch=1.1 is view=//depot/proj/1.1/... 1047 1048 @type p4port: string 1049 @param p4port: Specify the perforce server to connection in the format 1050 <host>:<port>. Example "perforce.example.com:1666" 1051 1052 @type p4user: string 1053 @param p4user: The perforce user to run the command as. 1054 1055 @type p4passwd: string 1056 @param p4passwd: The password for the perforce user. 1057 1058 @type p4extra_views: list of tuples 1059 @param p4extra_views: Extra views to be added to 1060 the client that is being used. 1061 1062 @type p4client: string 1063 @param p4client: The perforce client to use for this buildslave. 1064 """ 1065 1066 self.branch = defaultBranch 1067 Source.__init__(self, **kwargs) 1068 self.addFactoryArguments(p4base=p4base, 1069 defaultBranch=defaultBranch, 1070 p4port=p4port, 1071 p4user=p4user, 1072 p4passwd=p4passwd, 1073 p4extra_views=p4extra_views, 1074 p4client=p4client, 1075 ) 1076 self.args['p4port'] = p4port 1077 self.args['p4user'] = p4user 1078 self.args['p4passwd'] = p4passwd 1079 self.args['p4base'] = p4base 1080 self.args['p4extra_views'] = p4extra_views 1081 self.p4client = p4client

1082

1083 - def setBuild(self, build):

1084 Source.setBuild(self, build) 1085 self.args['p4client'] = self.p4client % { 1086 'slave': build.slavename, 1087 'builder': build.builder.name, 1088 }

1089

1090 - def computeSourceRevision(self, changes):

1091 if not changes: 1092 return None 1093 lastChange = max([int(c.revision) for c in changes]) 1094 return lastChange

1095

1096 - def startVC(self, branch, revision, patch):

1097 slavever = self.slaveVersion("p4") 1098 assert slavever, "slave is too old, does not know about p4" 1099 args = dict(self.args) 1100 args['branch'] = branch or self.branch 1101 args['revision'] = revision 1102 args['patch'] = patch 1103 cmd = LoggedRemoteCommand("p4", args) 1104 self.startCommand(cmd)

1105

1106 -class P4Sync(Source):

1107 """This is a partial solution for using a P4 source repository. You are 1108 required to manually set up each build slave with a useful P4 1109 environment, which means setting various per-slave environment variables, 1110 and creating a P4 client specification which maps the right files into 1111 the slave's working directory. Once you have done that, this step merely 1112 performs a 'p4 sync' to update that workspace with the newest files. 1113 1114 Each slave needs the following environment: 1115 1116 - PATH: the 'p4' binary must be on the slave's PATH 1117 - P4USER: each slave needs a distinct user account 1118 - P4CLIENT: each slave needs a distinct client specification 1119 1120 You should use 'p4 client' (?) to set up a client view spec which maps 1121 the desired files into $SLAVEBASE/$BUILDERBASE/source . 1122 """ 1123 1124 name = "p4sync" 1125

1126 - def __init__(self, p4port, p4user, p4passwd, p4client, **kwargs):

1127 assert kwargs['mode'] == "copy", "P4Sync can only be used in mode=copy" 1128 self.branch = None 1129 Source.__init__(self, **kwargs) 1130 self.addFactoryArguments(p4port=p4port, 1131 p4user=p4user, 1132 p4passwd=p4passwd, 1133 p4client=p4client, 1134 ) 1135 self.args['p4port'] = p4port 1136 self.args['p4user'] = p4user 1137 self.args['p4passwd'] = p4passwd 1138 self.args['p4client'] = p4client

1139

1140 - def computeSourceRevision(self, changes):

1141 if not changes: 1142 return None 1143 lastChange = max([int(c.revision) for c in changes]) 1144 return lastChange

1145

1146 - def startVC(self, branch, revision, patch):

1147 slavever = self.slaveVersion("p4sync") 1148 assert slavever, "slave is too old, does not know about p4" 1149 cmd = LoggedRemoteCommand("p4sync", self.args) 1150 self.startCommand(cmd)

1151

1152 -class Monotone(Source):

1153 """Check out a revision from a monotone server at 'server_addr', 1154 branch 'branch'. 'revision' specifies which revision id to check 1155 out. 1156 1157 This step will first create a local database, if necessary, and then pull 1158 the contents of the server into the database. Then it will do the 1159 checkout/update from this database.""" 1160 1161 name = "monotone" 1162

1163 - def __init__(self, server_addr, branch, db_path="monotone.db", 1164 monotone="monotone", 1165 **kwargs):

1166 Source.__init__(self, **kwargs) 1167 self.addFactoryArguments(server_addr=server_addr, 1168 branch=branch, 1169 db_path=db_path, 1170 monotone=monotone, 1171 ) 1172 self.args.update({"server_addr": server_addr, 1173 "branch": branch, 1174 "db_path": db_path, 1175 "monotone": monotone})

1176

1177 - def computeSourceRevision(self, changes):

1178 if not changes: 1179 return None 1180 return changes[-1].revision

1181

1182 - def startVC(self):

1183 slavever = self.slaveVersion("monotone") 1184 assert slavever, "slave is too old, does not know about monotone" 1185 cmd = LoggedRemoteCommand("monotone", self.args) 1186 self.startCommand(cmd)

1187

Source Code for Module buildbot.steps.source