buildbot.steps.source

1 # -*- test-case-name: buildbot.test.test_vc -*- 2 3 from warnings import warn 4 from email.Utils import formatdate 5 from twisted.python import log 6 from buildbot.process.buildstep import LoggingBuildStep, LoggedRemoteCommand 7 from buildbot.interfaces import BuildSlaveTooOldError 8 from buildbot.status.builder import SKIPPED 9 10

11 -class Source(LoggingBuildStep):

12 """This is a base class to generate a source tree in the buildslave. 13 Each version control system has a specialized subclass, and is expected 14 to override __init__ and implement computeSourceRevision() and 15 startVC(). The class as a whole builds up the self.args dictionary, then 16 starts a LoggedRemoteCommand with those arguments. 17 """ 18 19 # if the checkout fails, there's no point in doing anything else 20 haltOnFailure = True 21 flunkOnFailure = True 22 notReally = False 23 24 branch = None # the default branch, should be set in __init__ 25

26 - def __init__(self, workdir=None, mode='update', alwaysUseLatest=False, 27 timeout=20*60, retry=None, **kwargs):

28 """ 29 @type workdir: string 30 @param workdir: local directory (relative to the Builder's root) 31 where the tree should be placed 32 33 @type mode: string 34 @param mode: the kind of VC operation that is desired: 35 - 'update': specifies that the checkout/update should be 36 performed directly into the workdir. Each build is performed 37 in the same directory, allowing for incremental builds. This 38 minimizes disk space, bandwidth, and CPU time. However, it 39 may encounter problems if the build process does not handle 40 dependencies properly (if you must sometimes do a 'clean 41 build' to make sure everything gets compiled), or if source 42 files are deleted but generated files can influence test 43 behavior (e.g. python's .pyc files), or when source 44 directories are deleted but generated files prevent CVS from 45 removing them. When used with a patched checkout, from a 46 previous buildbot try for instance, it will try to "revert" 47 the changes first and will do a clobber if it is unable to 48 get a clean checkout. The behavior is SCM-dependent. 49 50 - 'copy': specifies that the source-controlled workspace 51 should be maintained in a separate directory (called the 52 'copydir'), using checkout or update as necessary. For each 53 build, a new workdir is created with a copy of the source 54 tree (rm -rf workdir; cp -R -P -p copydir workdir). This 55 doubles the disk space required, but keeps the bandwidth low 56 (update instead of a full checkout). A full 'clean' build 57 is performed each time. This avoids any generated-file 58 build problems, but is still occasionally vulnerable to 59 problems such as a CVS repository being manually rearranged 60 (causing CVS errors on update) which are not an issue with 61 a full checkout. 62 63 - 'clobber': specifies that the working directory should be 64 deleted each time, necessitating a full checkout for each 65 build. This insures a clean build off a complete checkout, 66 avoiding any of the problems described above, but is 67 bandwidth intensive, as the whole source tree must be 68 pulled down for each build. 69 70 - 'export': is like 'clobber', except that e.g. the 'cvs 71 export' command is used to create the working directory. 72 This command removes all VC metadata files (the 73 CVS/.svn/{arch} directories) from the tree, which is 74 sometimes useful for creating source tarballs (to avoid 75 including the metadata in the tar file). Not all VC systems 76 support export. 77 78 @type alwaysUseLatest: boolean 79 @param alwaysUseLatest: whether to always update to the most 80 recent available sources for this build. 81 82 Normally the Source step asks its Build for a list of all 83 Changes that are supposed to go into the build, then computes a 84 'source stamp' (revision number or timestamp) that will cause 85 exactly that set of changes to be present in the checked out 86 tree. This is turned into, e.g., 'cvs update -D timestamp', or 87 'svn update -r revnum'. If alwaysUseLatest=True, bypass this 88 computation and always update to the latest available sources 89 for each build. 90 91 The source stamp helps avoid a race condition in which someone 92 commits a change after the master has decided to start a build 93 but before the slave finishes checking out the sources. At best 94 this results in a build which contains more changes than the 95 buildmaster thinks it has (possibly resulting in the wrong 96 person taking the blame for any problems that result), at worst 97 is can result in an incoherent set of sources (splitting a 98 non-atomic commit) which may not build at all. 99 100 @type retry: tuple of ints (delay, repeats) (or None) 101 @param retry: if provided, VC update failures are re-attempted up 102 to REPEATS times, with DELAY seconds between each 103 attempt. Some users have slaves with poor connectivity 104 to their VC repository, and they say that up to 80% of 105 their build failures are due to transient network 106 failures that could be handled by simply retrying a 107 couple times. 108 109 """ 110 111 LoggingBuildStep.__init__(self, **kwargs) 112 self.addFactoryArguments(workdir=workdir, 113 mode=mode, 114 alwaysUseLatest=alwaysUseLatest, 115 timeout=timeout, 116 retry=retry, 117 ) 118 119 assert mode in ("update", "copy", "clobber", "export") 120 if retry: 121 delay, repeats = retry 122 assert isinstance(repeats, int) 123 assert repeats > 0 124 self.args = {'mode': mode, 125 'workdir': workdir, 126 'timeout': timeout, 127 'retry': retry, 128 'patch': None, # set during .start 129 } 130 self.alwaysUseLatest = alwaysUseLatest 131 132 # Compute defaults for descriptions: 133 description = ["updating"] 134 descriptionDone = ["update"] 135 if mode == "clobber": 136 description = ["checkout"] 137 # because checkingouting takes too much space 138 descriptionDone = ["checkout"] 139 elif mode == "export": 140 description = ["exporting"] 141 descriptionDone = ["export"] 142 self.description = description 143 self.descriptionDone = descriptionDone

144

145 - def setStepStatus(self, step_status):

146 LoggingBuildStep.setStepStatus(self, step_status)

147

148 - def setDefaultWorkdir(self, workdir):

149 self.args['workdir'] = self.args['workdir'] or workdir

150

151 - def describe(self, done=False):

152 if done: 153 return self.descriptionDone 154 return self.description

155

156 - def computeSourceRevision(self, changes):

157 """Each subclass must implement this method to do something more 158 precise than -rHEAD every time. For version control systems that use 159 repository-wide change numbers (SVN, P4), this can simply take the 160 maximum such number from all the changes involved in this build. For 161 systems that do not (CVS), it needs to create a timestamp based upon 162 the latest Change, the Build's treeStableTimer, and an optional 163 self.checkoutDelay value.""" 164 return None

165

166 - def computeRepositoryURL(self, repository):

167 ''' 168 Helper function that the repository URL based on the parameter the 169 source step took and the Change 'repository' property 170 ''' 171 172 assert not repository or callable(repository) or isinstance(repository, dict) or \ 173 isinstance(repository, str) or isinstance(repository, unicode) 174 175 s = self.build.getSourceStamp() 176 if not repository: 177 assert s.repository 178 return str(s.repository) 179 else: 180 if callable(repository): 181 return str(repository(s.repository)) 182 elif isinstance(repository, dict): 183 return str(repository.get(s.repository)) 184 else: # string or unicode 185 try: 186 repourl = str(repository % s.repository) 187 except TypeError: 188 # that's the backward compatibility case 189 repourl = repository 190 return str(repourl)

191

192 - def start(self):

193 if self.notReally: 194 log.msg("faking %s checkout/update" % self.name) 195 self.step_status.setText(["fake", self.name, "successful"]) 196 self.addCompleteLog("log", 197 "Faked %s checkout/update 'successful'\n" \ 198 % self.name) 199 return SKIPPED 200 201 # what source stamp would this build like to use? 202 s = self.build.getSourceStamp() 203 # if branch is None, then use the Step's "default" branch 204 branch = s.branch or self.branch 205 # if revision is None, use the latest sources (-rHEAD) 206 revision = s.revision 207 if not revision and not self.alwaysUseLatest: 208 revision = self.computeSourceRevision(s.changes) 209 # the revision property is currently None, so set it to something 210 # more interesting 211 self.setProperty('revision', str(revision), "Source") 212 213 # if patch is None, then do not patch the tree after checkout 214 215 # 'patch' is None or a tuple of (patchlevel, diff, root) 216 # root is optional. 217 patch = s.patch 218 if patch: 219 self.addCompleteLog("patch", patch[1]) 220 221 if self.alwaysUseLatest: 222 revision = None 223 self.startVC(branch, revision, patch)

224

225 - def commandComplete(self, cmd):

226 if cmd.updates.has_key("got_revision"): 227 got_revision = cmd.updates["got_revision"][-1] 228 if got_revision is not None: 229 self.setProperty("got_revision", str(got_revision), "Source")

230 231 232

233 -class BK(Source):

234 """I perform BitKeeper checkout/update operations.""" 235 236 name = 'bk' 237

238 - def __init__(self, bkurl=None, baseURL=None, 239 directory=None, extra_args=None, **kwargs):

240 """ 241 @type bkurl: string 242 @param bkurl: the URL which points to the BitKeeper server. 243 244 @type baseURL: string 245 @param baseURL: if branches are enabled, this is the base URL to 246 which a branch name will be appended. It should 247 probably end in a slash. Use exactly one of 248 C{bkurl} and C{baseURL}. 249 """ 250 251 self.bkurl = bkurl 252 self.baseURL = baseURL 253 self.extra_args = extra_args 254 255 Source.__init__(self, **kwargs) 256 self.addFactoryArguments(bkurl=bkurl, 257 baseURL=baseURL, 258 directory=directory, 259 extra_args=extra_args, 260 ) 261 262 if bkurl and baseURL: 263 raise ValueError("you must use exactly one of bkurl and baseURL")

264 265

266 - def computeSourceRevision(self, changes):

267 return changes.revision

268 269

270 - def startVC(self, branch, revision, patch):

271 272 warnings = [] 273 slavever = self.slaveVersion("bk") 274 if not slavever: 275 m = "slave does not have the 'bk' command" 276 raise BuildSlaveTooOldError(m) 277 278 if self.bkurl: 279 assert not branch # we need baseURL= to use branches 280 self.args['bkurl'] = self.computeRepositoryURL(self.bkurl) 281 else: 282 self.args['bkurl'] = self.computeRepositoryURL(self.baseURL) + branch 283 self.args['revision'] = revision 284 self.args['patch'] = patch 285 self.args['branch'] = branch 286 if self.extra_args is not None: 287 self.args['extra_args'] = self.extra_args 288 289 revstuff = [] 290 revstuff.append("[branch]") 291 if revision is not None: 292 revstuff.append("r%s" % revision) 293 if patch is not None: 294 revstuff.append("[patch]") 295 self.description.extend(revstuff) 296 self.descriptionDone.extend(revstuff) 297 298 cmd = LoggedRemoteCommand("bk", self.args) 299 self.startCommand(cmd, warnings)

300 301 302

303 -class CVS(Source):

304 """I do CVS checkout/update operations. 305 306 Note: if you are doing anonymous/pserver CVS operations, you will need 307 to manually do a 'cvs login' on each buildslave before the slave has any 308 hope of success. XXX: fix then, take a cvs password as an argument and 309 figure out how to do a 'cvs login' on each build 310 """ 311 312 name = "cvs" 313 314 #progressMetrics = ('output',) 315 # 316 # additional things to track: update gives one stderr line per directory 317 # (starting with 'cvs server: Updating ') (and is fairly stable if files 318 # is empty), export gives one line per directory (starting with 'cvs 319 # export: Updating ') and another line per file (starting with U). Would 320 # be nice to track these, requires grepping LogFile data for lines, 321 # parsing each line. Might be handy to have a hook in LogFile that gets 322 # called with each complete line. 323

324 - def __init__(self, cvsroot=None, cvsmodule="", 325 global_options=[], branch=None, checkoutDelay=None, 326 checkout_options=[], export_options=[], extra_options=[], 327 login=None, 328 **kwargs):

329 330 """ 331 @type cvsroot: string 332 @param cvsroot: CVS Repository from which the source tree should 333 be obtained. '/home/warner/Repository' for local 334 or NFS-reachable repositories, 335 ':pserver:anon@foo.com:/cvs' for anonymous CVS, 336 'user@host.com:/cvs' for non-anonymous CVS or 337 CVS over ssh. Lots of possibilities, check the 338 CVS documentation for more. 339 340 @type cvsmodule: string 341 @param cvsmodule: subdirectory of CVS repository that should be 342 retrieved 343 344 @type login: string or None 345 @param login: if not None, a string which will be provided as a 346 password to the 'cvs login' command, used when a 347 :pserver: method is used to access the repository. 348 This login is only needed once, but must be run 349 each time (just before the CVS operation) because 350 there is no way for the buildslave to tell whether 351 it was previously performed or not. 352 353 @type branch: string 354 @param branch: the default branch name, will be used in a '-r' 355 argument to specify which branch of the source tree 356 should be used for this checkout. Defaults to None, 357 which means to use 'HEAD'. 358 359 @type checkoutDelay: int or None 360 @param checkoutDelay: if not None, the number of seconds to put 361 between the last known Change and the 362 timestamp given to the -D argument. This 363 defaults to exactly half of the parent 364 Build's .treeStableTimer, but it could be 365 set to something else if your CVS change 366 notification has particularly weird 367 latency characteristics. 368 369 @type global_options: list of strings 370 @param global_options: these arguments are inserted in the cvs 371 command line, before the 372 'checkout'/'update' command word. See 373 'cvs --help-options' for a list of what 374 may be accepted here. ['-r'] will make 375 the checked out files read only. ['-r', 376 '-R'] will also assume the repository is 377 read-only (I assume this means it won't 378 use locks to insure atomic access to the 379 ,v files). 380 381 @type checkout_options: list of strings 382 @param checkout_options: these arguments are inserted in the cvs 383 command line, after 'checkout' but before 384 branch or revision specifiers. 385 386 @type export_options: list of strings 387 @param export_options: these arguments are inserted in the cvs 388 command line, after 'export' but before 389 branch or revision specifiers. 390 391 @type extra_options: list of strings 392 @param extra_options: these arguments are inserted in the cvs 393 command line, after 'checkout' or 'export' but before 394 branch or revision specifiers. 395 """ 396 397 self.checkoutDelay = checkoutDelay 398 self.branch = branch 399 self.cvsroot = cvsroot 400 401 Source.__init__(self, **kwargs) 402 self.addFactoryArguments(cvsroot=cvsroot, 403 cvsmodule=cvsmodule, 404 global_options=global_options, 405 checkout_options=checkout_options, 406 export_options=export_options, 407 extra_options=extra_options, 408 branch=branch, 409 checkoutDelay=checkoutDelay, 410 login=login, 411 ) 412 413 self.args.update({'cvsmodule': cvsmodule, 414 'global_options': global_options, 415 'checkout_options':checkout_options, 416 'export_options':export_options, 417 'extra_options':extra_options, 418 'login': login, 419 })

420

421 - def computeSourceRevision(self, changes):

422 if not changes: 423 return None 424 lastChange = max([c.when for c in changes]) 425 if self.checkoutDelay is not None: 426 when = lastChange + self.checkoutDelay 427 else: 428 lastSubmit = max([r.submittedAt for r in self.build.requests]) 429 when = (lastChange + lastSubmit) / 2 430 return formatdate(when)

431

432 - def startVC(self, branch, revision, patch):

433 if self.slaveVersionIsOlderThan("cvs", "1.39"): 434 # the slave doesn't know to avoid re-using the same sourcedir 435 # when the branch changes. We have no way of knowing which branch 436 # the last build used, so if we're using a non-default branch and 437 # either 'update' or 'copy' modes, it is safer to refuse to 438 # build, and tell the user they need to upgrade the buildslave. 439 if (branch != self.branch 440 and self.args['mode'] in ("update", "copy")): 441 m = ("This buildslave (%s) does not know about multiple " 442 "branches, and using mode=%s would probably build the " 443 "wrong tree. " 444 "Refusing to build. Please upgrade the buildslave to " 445 "buildbot-0.7.0 or newer." % (self.build.slavename, 446 self.args['mode'])) 447 log.msg(m) 448 raise BuildSlaveTooOldError(m) 449 450 if self.slaveVersionIsOlderThan("cvs", "2.10"): 451 if self.args['extra_options'] or self.args['export_options']: 452 m = ("This buildslave (%s) does not support export_options " 453 "or extra_options arguments to the CVS step." 454 % (self.build.slavename)) 455 log.msg(m) 456 raise BuildSlaveTooOldError(m) 457 # the unwanted args are empty, and will probably be ignored by 458 # the slave, but delete them just to be safe 459 del self.args['export_options'] 460 del self.args['extra_options'] 461 462 if branch is None: 463 branch = "HEAD" 464 self.args['cvsroot'] = self.computeRepositoryURL(self.cvsroot) 465 self.args['branch'] = branch 466 self.args['revision'] = revision 467 self.args['patch'] = patch 468 469 if self.args['branch'] == "HEAD" and self.args['revision']: 470 # special case. 'cvs update -r HEAD -D today' gives no files 471 # TODO: figure out why, see if it applies to -r BRANCH 472 self.args['branch'] = None 473 474 # deal with old slaves 475 warnings = [] 476 slavever = self.slaveVersion("cvs", "old") 477 478 if slavever == "old": 479 # 0.5.0 480 if self.args['mode'] == "export": 481 self.args['export'] = 1 482 elif self.args['mode'] == "clobber": 483 self.args['clobber'] = 1 484 elif self.args['mode'] == "copy": 485 self.args['copydir'] = "source" 486 self.args['tag'] = self.args['branch'] 487 assert not self.args['patch'] # 0.5.0 slave can't do patch 488 489 cmd = LoggedRemoteCommand("cvs", self.args) 490 self.startCommand(cmd, warnings)

491 492

493 -class SVN(Source):

494 """I perform Subversion checkout/update operations.""" 495 496 name = 'svn' 497

498 - def __init__(self, svnurl=None, baseURL=None, defaultBranch=None, 499 directory=None, username=None, password=None, 500 extra_args=None, keep_on_purge=None, ignore_ignores=None, 501 always_purge=None, depth=None, **kwargs):

502 """ 503 @type svnurl: string 504 @param svnurl: the URL which points to the Subversion server, 505 combining the access method (HTTP, ssh, local file), 506 the repository host/port, the repository path, the 507 sub-tree within the repository, and the branch to 508 check out. Use exactly one of C{svnurl} and C{baseURL}. 509 510 @param baseURL: if branches are enabled, this is the base URL to 511 which a branch name will be appended. It should 512 probably end in a slash. Use exactly one of 513 C{svnurl} and C{baseURL}. 514 515 @param defaultBranch: if branches are enabled, this is the branch 516 to use if the Build does not specify one 517 explicitly. It will simply be appended 518 to C{baseURL} and the result handed to 519 the SVN command. 520 521 @type username: string 522 @param username: username to pass to svn's --username 523 524 @type password: string 525 @param password: password to pass to svn's --password 526 """ 527 528 if not 'workdir' in kwargs and directory is not None: 529 # deal with old configs 530 warn("Please use workdir=, not directory=", DeprecationWarning) 531 kwargs['workdir'] = directory 532 533 self.svnurl = svnurl 534 self.baseURL = baseURL 535 self.branch = defaultBranch 536 self.username = username 537 self.password = password 538 self.extra_args = extra_args 539 self.keep_on_purge = keep_on_purge 540 self.ignore_ignores = ignore_ignores 541 self.always_purge = always_purge 542 self.depth = depth 543 544 Source.__init__(self, **kwargs) 545 self.addFactoryArguments(svnurl=svnurl, 546 baseURL=baseURL, 547 defaultBranch=defaultBranch, 548 directory=directory, 549 username=username, 550 password=password, 551 extra_args=extra_args, 552 keep_on_purge=keep_on_purge, 553 ignore_ignores=ignore_ignores, 554 always_purge=always_purge, 555 depth=depth, 556 ) 557 558 if svnurl and baseURL: 559 raise ValueError("you must use either svnurl OR baseURL")

560

561 - def computeSourceRevision(self, changes):

562 if not changes or None in [c.revision for c in changes]: 563 return None 564 lastChange = max([int(c.revision) for c in changes]) 565 return lastChange

566

567 - def startVC(self, branch, revision, patch):

568 569 # handle old slaves 570 warnings = [] 571 slavever = self.slaveVersion("svn", "old") 572 if not slavever: 573 m = "slave does not have the 'svn' command" 574 raise BuildSlaveTooOldError(m) 575 576 if self.slaveVersionIsOlderThan("svn", "1.39"): 577 # the slave doesn't know to avoid re-using the same sourcedir 578 # when the branch changes. We have no way of knowing which branch 579 # the last build used, so if we're using a non-default branch and 580 # either 'update' or 'copy' modes, it is safer to refuse to 581 # build, and tell the user they need to upgrade the buildslave. 582 if (branch != self.branch 583 and self.args['mode'] in ("update", "copy")): 584 m = ("This buildslave (%s) does not know about multiple " 585 "branches, and using mode=%s would probably build the " 586 "wrong tree. " 587 "Refusing to build. Please upgrade the buildslave to " 588 "buildbot-0.7.0 or newer." % (self.build.slavename, 589 self.args['mode'])) 590 raise BuildSlaveTooOldError(m) 591 592 if slavever == "old": 593 # 0.5.0 compatibility 594 if self.args['mode'] in ("clobber", "copy"): 595 # TODO: use some shell commands to make up for the 596 # deficiency, by blowing away the old directory first (thus 597 # forcing a full checkout) 598 warnings.append("WARNING: this slave can only do SVN updates" 599 ", not mode=%s\n" % self.args['mode']) 600 log.msg("WARNING: this slave only does mode=update") 601 if self.args['mode'] == "export": 602 raise BuildSlaveTooOldError("old slave does not have " 603 "mode=export") 604 self.args['directory'] = self.args['workdir'] 605 if revision is not None: 606 # 0.5.0 can only do HEAD. We have no way of knowing whether 607 # the requested revision is HEAD or not, and for 608 # slowly-changing trees this will probably do the right 609 # thing, so let it pass with a warning 610 m = ("WARNING: old slave can only update to HEAD, not " 611 "revision=%s" % revision) 612 log.msg(m) 613 warnings.append(m + "\n") 614 revision = "HEAD" # interprets this key differently 615 if patch: 616 raise BuildSlaveTooOldError("old slave can't do patch") 617 618 if self.svnurl: 619 self.args['svnurl'] = self.computeRepositoryURL(self.svnurl) 620 else: 621 self.args['svnurl'] = (self.computeRepositoryURL(self.baseURL) + 622 branch) 623 self.args['revision'] = revision 624 self.args['patch'] = patch 625 626 self.args['always_purge'] = self.always_purge 627 628 #Set up depth if specified 629 if self.depth is not None: 630 if self.slaveVersionIsOlderThan("svn","2.9"): 631 m = ("This buildslave (%s) does not support svn depth " 632 "arguments. Refusing to build. " 633 "Please upgrade the buildslave." % (self.build.slavename)) 634 raise BuildSlaveTooOldError(m) 635 else: 636 self.args['depth'] = self.depth 637 638 if self.username is not None or self.password is not None: 639 if self.slaveVersionIsOlderThan("svn", "2.8"): 640 m = ("This buildslave (%s) does not support svn usernames " 641 "and passwords. " 642 "Refusing to build. Please upgrade the buildslave to " 643 "buildbot-0.7.10 or newer." % (self.build.slavename,)) 644 raise BuildSlaveTooOldError(m) 645 if self.username is not None: 646 self.args['username'] = self.username 647 if self.password is not None: 648 self.args['password'] = self.password 649 650 if self.extra_args is not None: 651 self.args['extra_args'] = self.extra_args 652 653 revstuff = [] 654 #revstuff.append(self.args['svnurl']) 655 if self.args['svnurl'].find('trunk') == -1: 656 revstuff.append("[branch]") 657 if revision is not None: 658 revstuff.append("r%s" % revision) 659 if patch is not None: 660 revstuff.append("[patch]") 661 self.description.extend(revstuff) 662 self.descriptionDone.extend(revstuff) 663 664 cmd = LoggedRemoteCommand("svn", self.args) 665 self.startCommand(cmd, warnings)

666 667

668 -class Darcs(Source):

669 """Check out a source tree from a Darcs repository at 'repourl'. 670 671 Darcs has no concept of file modes. This means the eXecute-bit will be 672 cleared on all source files. As a result, you may need to invoke 673 configuration scripts with something like: 674 675 C{s(step.Configure, command=['/bin/sh', './configure'])} 676 """ 677 678 name = "darcs" 679

680 - def __init__(self, repourl=None, baseURL=None, defaultBranch=None, 681 **kwargs):

682 """ 683 @type repourl: string 684 @param repourl: the URL which points at the Darcs repository. This 685 is used as the default branch. Using C{repourl} does 686 not enable builds of alternate branches: use 687 C{baseURL} to enable this. Use either C{repourl} or 688 C{baseURL}, not both. 689 690 @param baseURL: if branches are enabled, this is the base URL to 691 which a branch name will be appended. It should 692 probably end in a slash. Use exactly one of 693 C{repourl} and C{baseURL}. 694 695 @param defaultBranch: if branches are enabled, this is the branch 696 to use if the Build does not specify one 697 explicitly. It will simply be appended to 698 C{baseURL} and the result handed to the 699 'darcs pull' command. 700 """ 701 self.repourl = repourl 702 self.baseURL = baseURL 703 self.branch = defaultBranch 704 Source.__init__(self, **kwargs) 705 self.addFactoryArguments(repourl=repourl, 706 baseURL=baseURL, 707 defaultBranch=defaultBranch, 708 ) 709 assert self.args['mode'] != "export", \ 710 "Darcs does not have an 'export' mode" 711 if repourl and baseURL: 712 raise ValueError("you must provide exactly one of repourl and" 713 " baseURL")

714

715 - def startVC(self, branch, revision, patch):

716 slavever = self.slaveVersion("darcs") 717 if not slavever: 718 m = "slave is too old, does not know about darcs" 719 raise BuildSlaveTooOldError(m) 720 721 if self.slaveVersionIsOlderThan("darcs", "1.39"): 722 if revision: 723 # TODO: revisit this once we implement computeSourceRevision 724 m = "0.6.6 slaves can't handle args['revision']" 725 raise BuildSlaveTooOldError(m) 726 727 # the slave doesn't know to avoid re-using the same sourcedir 728 # when the branch changes. We have no way of knowing which branch 729 # the last build used, so if we're using a non-default branch and 730 # either 'update' or 'copy' modes, it is safer to refuse to 731 # build, and tell the user they need to upgrade the buildslave. 732 if (branch != self.branch 733 and self.args['mode'] in ("update", "copy")): 734 m = ("This buildslave (%s) does not know about multiple " 735 "branches, and using mode=%s would probably build the " 736 "wrong tree. " 737 "Refusing to build. Please upgrade the buildslave to " 738 "buildbot-0.7.0 or newer." % (self.build.slavename, 739 self.args['mode'])) 740 raise BuildSlaveTooOldError(m) 741 742 if self.repourl: 743 assert not branch # we need baseURL= to use branches 744 self.args['repourl'] = self.computeRepositoryURL(self.repourl) 745 else: 746 self.args['repourl'] = self.computeRepositoryURL(self.baseURL) + branch 747 self.args['revision'] = revision 748 self.args['patch'] = patch 749 750 revstuff = [] 751 if branch is not None and branch != self.branch: 752 revstuff.append("[branch]") 753 self.description.extend(revstuff) 754 self.descriptionDone.extend(revstuff) 755 756 cmd = LoggedRemoteCommand("darcs", self.args) 757 self.startCommand(cmd)

758 759

760 -class Git(Source):

761 """Check out a source tree from a git repository 'repourl'.""" 762 763 name = "git" 764

765 - def __init__(self, repourl=None, 766 branch="master", 767 submodules=False, 768 ignore_ignores=None, 769 shallow=False, 770 **kwargs):

771 """ 772 @type repourl: string 773 @param repourl: the URL which points at the git repository 774 775 @type branch: string 776 @param branch: The branch or tag to check out by default. If 777 a build specifies a different branch, it will 778 be used instead of this. 779 780 @type submodules: boolean 781 @param submodules: Whether or not to update (and initialize) 782 git submodules. 783 784 @type shallow: boolean 785 @param shallow: Use a shallow or clone, if possible 786 """ 787 Source.__init__(self, **kwargs) 788 self.repourl = repourl 789 self.addFactoryArguments(repourl=repourl, 790 branch=branch, 791 submodules=submodules, 792 ignore_ignores=ignore_ignores, 793 shallow=shallow, 794 ) 795 self.args.update({'branch': branch, 796 'submodules': submodules, 797 'ignore_ignores': ignore_ignores, 798 'shallow': shallow, 799 })

800

801 - def computeSourceRevision(self, changes):

802 if not changes: 803 return None 804 return changes[-1].revision

805

806 - def startVC(self, branch, revision, patch):

807 if branch is not None: 808 self.args['branch'] = branch 809 810 self.args['repourl'] = self.computeRepositoryURL(self.repourl) 811 self.args['revision'] = revision 812 self.args['patch'] = patch 813 slavever = self.slaveVersion("git") 814 if not slavever: 815 raise BuildSlaveTooOldError("slave is too old, does not know " 816 "about git") 817 cmd = LoggedRemoteCommand("git", self.args) 818 self.startCommand(cmd)

819 820

821 -class Arch(Source):

822 """Check out a source tree from an Arch repository named 'archive' 823 available at 'url'. 'version' specifies which version number (development 824 line) will be used for the checkout: this is mostly equivalent to a 825 branch name. This version uses the 'tla' tool to do the checkout, to use 826 'baz' see L{Bazaar} instead. 827 """ 828 829 name = "arch" 830 # TODO: slaves >0.6.6 will accept args['build-config'], so use it 831

832 - def __init__(self, url=None, version=None, archive=None, **kwargs):

833 """ 834 @type url: string 835 @param url: the Arch coordinates of the repository. This is 836 typically an http:// URL, but could also be the absolute 837 pathname of a local directory instead. 838 839 @type version: string 840 @param version: the category--branch--version to check out. This is 841 the default branch. If a build specifies a different 842 branch, it will be used instead of this. 843 844 @type archive: string 845 @param archive: The archive name. If provided, it must match the one 846 that comes from the repository. If not, the 847 repository's default will be used. 848 """ 849 warn("Support for Arch will be removed in 0.8.2", DeprecationWarning) 850 self.branch = version 851 self.url = url 852 Source.__init__(self, **kwargs) 853 self.addFactoryArguments(url=url, 854 version=version, 855 archive=archive, 856 ) 857 assert version, "version should be provided" 858 self.args.update({'archive': archive})

859

860 - def computeSourceRevision(self, changes):

861 # in Arch, fully-qualified revision numbers look like: 862 # arch@buildbot.sourceforge.net--2004/buildbot--dev--0--patch-104 863 # For any given builder, all of this is fixed except the patch-104. 864 # The Change might have any part of the fully-qualified string, so we 865 # just look for the last part. We return the "patch-NN" string. 866 if not changes: 867 return None 868 lastChange = None 869 for c in changes: 870 if not c.revision: 871 continue 872 if c.revision.endswith("--base-0"): 873 rev = 0 874 else: 875 i = c.revision.rindex("patch") 876 rev = int(c.revision[i+len("patch-"):]) 877 lastChange = max(lastChange, rev) 878 if lastChange is None: 879 return None 880 if lastChange == 0: 881 return "base-0" 882 return "patch-%d" % lastChange

883

884 - def checkSlaveVersion(self, cmd, branch):

885 warnings = [] 886 slavever = self.slaveVersion(cmd) 887 if not slavever: 888 m = "slave is too old, does not know about %s" % cmd 889 raise BuildSlaveTooOldError(m) 890 891 # slave 1.28 and later understand 'revision' 892 if self.slaveVersionIsOlderThan(cmd, "1.28"): 893 if not self.alwaysUseLatest: 894 # we don't know whether our requested revision is the latest 895 # or not. If the tree does not change very quickly, this will 896 # probably build the right thing, so emit a warning rather 897 # than refuse to build at all 898 m = "WARNING, buildslave is too old to use a revision" 899 log.msg(m) 900 warnings.append(m + "\n") 901 902 if self.slaveVersionIsOlderThan(cmd, "1.39"): 903 # the slave doesn't know to avoid re-using the same sourcedir 904 # when the branch changes. We have no way of knowing which branch 905 # the last build used, so if we're using a non-default branch and 906 # either 'update' or 'copy' modes, it is safer to refuse to 907 # build, and tell the user they need to upgrade the buildslave. 908 if (branch != self.branch 909 and self.args['mode'] in ("update", "copy")): 910 m = ("This buildslave (%s) does not know about multiple " 911 "branches, and using mode=%s would probably build the " 912 "wrong tree. " 913 "Refusing to build. Please upgrade the buildslave to " 914 "buildbot-0.7.0 or newer." % (self.build.slavename, 915 self.args['mode'])) 916 log.msg(m) 917 raise BuildSlaveTooOldError(m) 918 919 return warnings

920

921 - def startVC(self, branch, revision, patch):

922 self.args['url'] = self.computeRepositoryURL(self.url), 923 self.args['version'] = branch 924 self.args['revision'] = revision 925 self.args['patch'] = patch 926 warnings = self.checkSlaveVersion("arch", branch) 927 928 revstuff = [] 929 if branch is not None and branch != self.branch: 930 revstuff.append("[branch]") 931 if revision is not None: 932 revstuff.append("patch%s" % revision) 933 self.description.extend(revstuff) 934 self.descriptionDone.extend(revstuff) 935 936 cmd = LoggedRemoteCommand("arch", self.args) 937 self.startCommand(cmd, warnings)

938 939

940 -class Bazaar(Arch):

941 """Bazaar is an alternative client for Arch repositories. baz is mostly 942 compatible with tla, but archive registration is slightly different.""" 943 944 # TODO: slaves >0.6.6 will accept args['build-config'], so use it 945

946 - def __init__(self, url, version, archive, **kwargs):

947 """ 948 @type url: string 949 @param url: the Arch coordinates of the repository. This is 950 typically an http:// URL, but could also be the absolute 951 pathname of a local directory instead. 952 953 @type version: string 954 @param version: the category--branch--version to check out 955 956 @type archive: string 957 @param archive: The archive name (required). This must always match 958 the one that comes from the repository, otherwise the 959 buildslave will attempt to get sources from the wrong 960 archive. 961 """ 962 self.branch = version 963 self.url = url 964 Source.__init__(self, **kwargs) 965 self.addFactoryArguments(url=url, 966 version=version, 967 archive=archive, 968 ) 969 self.args.update({'archive': archive, 970 })

971

972 - def startVC(self, branch, revision, patch):

973 self.args['url'] = self.computeRepositoryURL(url), 974 self.args['version'] = branch 975 self.args['revision'] = revision 976 self.args['patch'] = patch 977 warnings = self.checkSlaveVersion("bazaar", branch) 978 979 revstuff = [] 980 if branch is not None and branch != self.branch: 981 revstuff.append("[branch]") 982 if revision is not None: 983 revstuff.append("patch%s" % revision) 984 self.description.extend(revstuff) 985 self.descriptionDone.extend(revstuff) 986 987 cmd = LoggedRemoteCommand("bazaar", self.args) 988 self.startCommand(cmd, warnings)

989

990 -class Bzr(Source):

991 """Check out a source tree from a bzr (Bazaar) repository at 'repourl'. 992 993 """ 994 995 name = "bzr" 996

997 - def __init__(self, repourl=None, baseURL=None, defaultBranch=None, 998 forceSharedRepo=None, 999 **kwargs):

1000 """ 1001 @type repourl: string 1002 @param repourl: the URL which points at the bzr repository. This 1003 is used as the default branch. Using C{repourl} does 1004 not enable builds of alternate branches: use 1005 C{baseURL} to enable this. Use either C{repourl} or 1006 C{baseURL}, not both. 1007 1008 @param baseURL: if branches are enabled, this is the base URL to 1009 which a branch name will be appended. It should 1010 probably end in a slash. Use exactly one of 1011 C{repourl} and C{baseURL}. 1012 1013 @param defaultBranch: if branches are enabled, this is the branch 1014 to use if the Build does not specify one 1015 explicitly. It will simply be appended to 1016 C{baseURL} and the result handed to the 1017 'bzr checkout pull' command. 1018 1019 1020 @param forceSharedRepo: Boolean, defaults to False. If set to True, 1021 the working directory will be made into a 1022 bzr shared repository if it is not already. 1023 Shared repository greatly reduces the amount 1024 of history data that needs to be downloaded 1025 if not using update/copy mode, or if using 1026 update/copy mode with multiple branches. 1027 """ 1028 self.repourl = repourl 1029 self.baseURL = baseURL 1030 self.branch = defaultBranch 1031 Source.__init__(self, **kwargs) 1032 self.addFactoryArguments(repourl=repourl, 1033 baseURL=baseURL, 1034 defaultBranch=defaultBranch, 1035 forceSharedRepo=forceSharedRepo 1036 ) 1037 self.args.update({'forceSharedRepo': forceSharedRepo}) 1038 if repourl and baseURL: 1039 raise ValueError("you must provide exactly one of repourl and" 1040 " baseURL")

1041

1042 - def computeSourceRevision(self, changes):

1043 if not changes: 1044 return None 1045 lastChange = max([int(c.revision) for c in changes]) 1046 return lastChange

1047

1048 - def startVC(self, branch, revision, patch):

1049 slavever = self.slaveVersion("bzr") 1050 if not slavever: 1051 m = "slave is too old, does not know about bzr" 1052 raise BuildSlaveTooOldError(m) 1053 1054 if self.repourl: 1055 assert not branch # we need baseURL= to use branches 1056 self.args['repourl'] = self.computeRepositoryURL(self.repourl) 1057 else: 1058 self.args['repourl'] = self.computeRepositoryURL(self.baseURL) + branch 1059 self.args['revision'] = revision 1060 self.args['patch'] = patch 1061 1062 revstuff = [] 1063 if branch is not None and branch != self.branch: 1064 revstuff.append("[" + branch + "]") 1065 if revision is not None: 1066 revstuff.append("r%s" % revision) 1067 self.description.extend(revstuff) 1068 self.descriptionDone.extend(revstuff) 1069 1070 cmd = LoggedRemoteCommand("bzr", self.args) 1071 self.startCommand(cmd)

1072 1073

1074 -class Mercurial(Source):

1075 """Check out a source tree from a mercurial repository 'repourl'.""" 1076 1077 name = "hg" 1078

1079 - def __init__(self, repourl=None, baseURL=None, defaultBranch=None, 1080 branchType='dirname', clobberOnBranchChange=True, **kwargs):

1081 """ 1082 @type repourl: string 1083 @param repourl: the URL which points at the Mercurial repository. 1084 This uses the 'default' branch unless defaultBranch is 1085 specified below and the C{branchType} is set to 1086 'inrepo'. It is an error to specify a branch without 1087 setting the C{branchType} to 'inrepo'. 1088 1089 @param baseURL: if 'dirname' branches are enabled, this is the base URL 1090 to which a branch name will be appended. It should 1091 probably end in a slash. Use exactly one of C{repourl} 1092 and C{baseURL}. 1093 1094 @param defaultBranch: if branches are enabled, this is the branch 1095 to use if the Build does not specify one 1096 explicitly. 1097 For 'dirname' branches, It will simply be 1098 appended to C{baseURL} and the result handed to 1099 the 'hg update' command. 1100 For 'inrepo' branches, this specifies the named 1101 revision to which the tree will update after a 1102 clone. 1103 1104 @param branchType: either 'dirname' or 'inrepo' depending on whether 1105 the branch name should be appended to the C{baseURL} 1106 or the branch is a mercurial named branch and can be 1107 found within the C{repourl} 1108 1109 @param clobberOnBranchChange: boolean, defaults to True. If set and 1110 using inrepos branches, clobber the tree 1111 at each branch change. Otherwise, just 1112 update to the branch. 1113 """ 1114 self.repourl = repourl 1115 self.baseURL = baseURL 1116 self.branch = defaultBranch 1117 self.branchType = branchType 1118 self.clobberOnBranchChange = clobberOnBranchChange 1119 Source.__init__(self, **kwargs) 1120 self.addFactoryArguments(repourl=repourl, 1121 baseURL=baseURL, 1122 defaultBranch=defaultBranch, 1123 branchType=branchType, 1124 clobberOnBranchChange=clobberOnBranchChange, 1125 ) 1126 if repourl and baseURL: 1127 raise ValueError("you must provide exactly one of repourl and" 1128 " baseURL")

1129

1130 - def startVC(self, branch, revision, patch):

1131 slavever = self.slaveVersion("hg") 1132 if not slavever: 1133 raise BuildSlaveTooOldError("slave is too old, does not know " 1134 "about hg") 1135 1136 if self.repourl: 1137 # we need baseURL= to use dirname branches 1138 assert self.branchType == 'inrepo' or not branch 1139 self.args['repourl'] = self.computeRepositoryURL(self.repourl) 1140 if branch: 1141 self.args['branch'] = branch 1142 else: 1143 self.args['repourl'] = self.computeRepositoryURL(self.baseURL) + (branch or '') 1144 self.args['revision'] = revision 1145 self.args['patch'] = patch 1146 self.args['clobberOnBranchChange'] = self.clobberOnBranchChange 1147 self.args['branchType'] = self.branchType 1148 1149 revstuff = [] 1150 if branch is not None and branch != self.branch: 1151 revstuff.append("[branch]") 1152 self.description.extend(revstuff) 1153 self.descriptionDone.extend(revstuff) 1154 1155 cmd = LoggedRemoteCommand("hg", self.args) 1156 self.startCommand(cmd)

1157

1158 - def computeSourceRevision(self, changes):

1159 if not changes: 1160 return None 1161 # without knowing the revision ancestry graph, we can't sort the 1162 # changes at all. So for now, assume they were given to us in sorted 1163 # order, and just pay attention to the last one. See ticket #103 for 1164 # more details. 1165 if len(changes) > 1: 1166 log.msg("Mercurial.computeSourceRevision: warning: " 1167 "there are %d changes here, assuming the last one is " 1168 "the most recent" % len(changes)) 1169 return changes[-1].revision

1170 1171

1172 -class P4(Source):

1173 """ P4 is a class for accessing perforce revision control""" 1174 name = "p4" 1175

1176 - def __init__(self, p4base=None, defaultBranch=None, p4port=None, p4user=None, 1177 p4passwd=None, p4extra_views=[], 1178 p4client='buildbot_%(slave)s_%(builder)s', **kwargs):

1179 """ 1180 @type p4base: string 1181 @param p4base: A view into a perforce depot, typically 1182 "//depot/proj/" 1183 1184 @type defaultBranch: string 1185 @param defaultBranch: Identify a branch to build by default. Perforce 1186 is a view based branching system. So, the branch 1187 is normally the name after the base. For example, 1188 branch=1.0 is view=//depot/proj/1.0/... 1189 branch=1.1 is view=//depot/proj/1.1/... 1190 1191 @type p4port: string 1192 @param p4port: Specify the perforce server to connection in the format 1193 <host>:<port>. Example "perforce.example.com:1666" 1194 1195 @type p4user: string 1196 @param p4user: The perforce user to run the command as. 1197 1198 @type p4passwd: string 1199 @param p4passwd: The password for the perforce user. 1200 1201 @type p4extra_views: list of tuples 1202 @param p4extra_views: Extra views to be added to 1203 the client that is being used. 1204 1205 @type p4client: string 1206 @param p4client: The perforce client to use for this buildslave. 1207 """ 1208 1209 self.p4base = p4base 1210 self.branch = defaultBranch 1211 Source.__init__(self, **kwargs) 1212 self.addFactoryArguments(p4base=p4base, 1213 defaultBranch=defaultBranch, 1214 p4port=p4port, 1215 p4user=p4user, 1216 p4passwd=p4passwd, 1217 p4extra_views=p4extra_views, 1218 p4client=p4client, 1219 ) 1220 self.args['p4port'] = p4port 1221 self.args['p4user'] = p4user 1222 self.args['p4passwd'] = p4passwd 1223 self.args['p4extra_views'] = p4extra_views 1224 self.p4client = p4client

1225

1226 - def setBuild(self, build):

1227 Source.setBuild(self, build) 1228 self.args['p4base'] = self.computeRepositoryURL(self.p4base) 1229 self.args['p4client'] = self.p4client % { 1230 'slave': build.slavename, 1231 'builder': build.builder.name, 1232 }

1233

1234 - def computeSourceRevision(self, changes):

1235 if not changes: 1236 return None 1237 lastChange = max([int(c.revision) for c in changes]) 1238 return lastChange

1239

1240 - def startVC(self, branch, revision, patch):

1241 slavever = self.slaveVersion("p4") 1242 assert slavever, "slave is too old, does not know about p4" 1243 args = dict(self.args) 1244 args['branch'] = branch or self.branch 1245 args['revision'] = revision 1246 args['patch'] = patch 1247 cmd = LoggedRemoteCommand("p4", args) 1248 self.startCommand(cmd)

1249

1250 -class P4Sync(Source):

1251 """This is a partial solution for using a P4 source repository. You are 1252 required to manually set up each build slave with a useful P4 1253 environment, which means setting various per-slave environment variables, 1254 and creating a P4 client specification which maps the right files into 1255 the slave's working directory. Once you have done that, this step merely 1256 performs a 'p4 sync' to update that workspace with the newest files. 1257 1258 Each slave needs the following environment: 1259 1260 - PATH: the 'p4' binary must be on the slave's PATH 1261 - P4USER: each slave needs a distinct user account 1262 - P4CLIENT: each slave needs a distinct client specification 1263 1264 You should use 'p4 client' (?) to set up a client view spec which maps 1265 the desired files into $SLAVEBASE/$BUILDERBASE/source . 1266 """ 1267 1268 name = "p4sync" 1269

1270 - def __init__(self, p4port, p4user, p4passwd, p4client, **kwargs):

1271 assert kwargs['mode'] == "copy", "P4Sync can only be used in mode=copy" 1272 self.branch = None 1273 Source.__init__(self, **kwargs) 1274 self.addFactoryArguments(p4port=p4port, 1275 p4user=p4user, 1276 p4passwd=p4passwd, 1277 p4client=p4client, 1278 ) 1279 self.args['p4port'] = p4port 1280 self.args['p4user'] = p4user 1281 self.args['p4passwd'] = p4passwd 1282 self.args['p4client'] = p4client

1283

1284 - def computeSourceRevision(self, changes):

1285 if not changes: 1286 return None 1287 lastChange = max([int(c.revision) for c in changes]) 1288 return lastChange

1289

1290 - def startVC(self, branch, revision, patch):

1291 slavever = self.slaveVersion("p4sync") 1292 assert slavever, "slave is too old, does not know about p4" 1293 cmd = LoggedRemoteCommand("p4sync", self.args) 1294 self.startCommand(cmd)

1295

1296 -class Monotone(Source):

1297 """Check out a revision from a monotone server at 'server_addr', 1298 branch 'branch'. 'revision' specifies which revision id to check 1299 out. 1300 1301 This step will first create a local database, if necessary, and then pull 1302 the contents of the server into the database. Then it will do the 1303 checkout/update from this database.""" 1304 1305 name = "monotone" 1306

1307 - def __init__(self, server_addr=None, branch=None, db_path="monotone.db", 1308 monotone="monotone", 1309 **kwargs):

1310 warn("Support for Monotone will be removed in 0.8.2", DeprecationWarning) 1311 Source.__init__(self, **kwargs) 1312 self.addFactoryArguments(server_addr=server_addr, 1313 branch=branch, 1314 db_path=db_path, 1315 monotone=monotone, 1316 ) 1317 assert branch, "branch must be specified" 1318 self.args.update({"branch": branch, 1319 "db_path": db_path, 1320 "monotone": monotone})

1321

1322 - def computeSourceRevision(self, changes):

1323 if not changes: 1324 return None 1325 return changes[-1].revision

1326

1327 - def startVC(self):

1328 self.args["server_addr"] = self.computeRepositoryURL(server_addr), 1329 slavever = self.slaveVersion("monotone") 1330 assert slavever, "slave is too old, does not know about monotone" 1331 cmd = LoggedRemoteCommand("monotone", self.args) 1332 self.startCommand(cmd)

1333

Source Code for Module buildbot.steps.source