buildbot.steps.source

1 # -*- test-case-name: buildbot.test.test_vc -*- 2 3 from warnings import warn 4 from email.Utils import formatdate 5 from twisted.python import log 6 from buildbot.process.buildstep import LoggingBuildStep, LoggedRemoteCommand 7 from buildbot.interfaces import BuildSlaveTooOldError 8 from buildbot.status.builder import SKIPPED 9 10

11 -class Source(LoggingBuildStep):

12 """This is a base class to generate a source tree in the buildslave. 13 Each version control system has a specialized subclass, and is expected 14 to override __init__ and implement computeSourceRevision() and 15 startVC(). The class as a whole builds up the self.args dictionary, then 16 starts a LoggedRemoteCommand with those arguments. 17 """ 18 19 # if the checkout fails, there's no point in doing anything else 20 haltOnFailure = True 21 flunkOnFailure = True 22 notReally = False 23 24 branch = None # the default branch, should be set in __init__ 25

26 - def __init__(self, workdir=None, mode='update', alwaysUseLatest=False, 27 timeout=20*60, retry=None, **kwargs):

28 """ 29 @type workdir: string 30 @param workdir: local directory (relative to the Builder's root) 31 where the tree should be placed 32 33 @type mode: string 34 @param mode: the kind of VC operation that is desired: 35 - 'update': specifies that the checkout/update should be 36 performed directly into the workdir. Each build is performed 37 in the same directory, allowing for incremental builds. This 38 minimizes disk space, bandwidth, and CPU time. However, it 39 may encounter problems if the build process does not handle 40 dependencies properly (if you must sometimes do a 'clean 41 build' to make sure everything gets compiled), or if source 42 files are deleted but generated files can influence test 43 behavior (e.g. python's .pyc files), or when source 44 directories are deleted but generated files prevent CVS from 45 removing them. When used with a patched checkout, from a 46 previous buildbot try for instance, it will try to "revert" 47 the changes first and will do a clobber if it is unable to 48 get a clean checkout. The behavior is SCM-dependent. 49 50 - 'copy': specifies that the source-controlled workspace 51 should be maintained in a separate directory (called the 52 'copydir'), using checkout or update as necessary. For each 53 build, a new workdir is created with a copy of the source 54 tree (rm -rf workdir; cp -R -P -p copydir workdir). This 55 doubles the disk space required, but keeps the bandwidth low 56 (update instead of a full checkout). A full 'clean' build 57 is performed each time. This avoids any generated-file 58 build problems, but is still occasionally vulnerable to 59 problems such as a CVS repository being manually rearranged 60 (causing CVS errors on update) which are not an issue with 61 a full checkout. 62 63 - 'clobber': specifies that the working directory should be 64 deleted each time, necessitating a full checkout for each 65 build. This insures a clean build off a complete checkout, 66 avoiding any of the problems described above, but is 67 bandwidth intensive, as the whole source tree must be 68 pulled down for each build. 69 70 - 'export': is like 'clobber', except that e.g. the 'cvs 71 export' command is used to create the working directory. 72 This command removes all VC metadata files (the 73 CVS/.svn/{arch} directories) from the tree, which is 74 sometimes useful for creating source tarballs (to avoid 75 including the metadata in the tar file). Not all VC systems 76 support export. 77 78 @type alwaysUseLatest: boolean 79 @param alwaysUseLatest: whether to always update to the most 80 recent available sources for this build. 81 82 Normally the Source step asks its Build for a list of all 83 Changes that are supposed to go into the build, then computes a 84 'source stamp' (revision number or timestamp) that will cause 85 exactly that set of changes to be present in the checked out 86 tree. This is turned into, e.g., 'cvs update -D timestamp', or 87 'svn update -r revnum'. If alwaysUseLatest=True, bypass this 88 computation and always update to the latest available sources 89 for each build. 90 91 The source stamp helps avoid a race condition in which someone 92 commits a change after the master has decided to start a build 93 but before the slave finishes checking out the sources. At best 94 this results in a build which contains more changes than the 95 buildmaster thinks it has (possibly resulting in the wrong 96 person taking the blame for any problems that result), at worst 97 is can result in an incoherent set of sources (splitting a 98 non-atomic commit) which may not build at all. 99 100 @type retry: tuple of ints (delay, repeats) (or None) 101 @param retry: if provided, VC update failures are re-attempted up 102 to REPEATS times, with DELAY seconds between each 103 attempt. Some users have slaves with poor connectivity 104 to their VC repository, and they say that up to 80% of 105 their build failures are due to transient network 106 failures that could be handled by simply retrying a 107 couple times. 108 109 """ 110 111 LoggingBuildStep.__init__(self, **kwargs) 112 self.addFactoryArguments(workdir=workdir, 113 mode=mode, 114 alwaysUseLatest=alwaysUseLatest, 115 timeout=timeout, 116 retry=retry, 117 ) 118 119 assert mode in ("update", "copy", "clobber", "export") 120 if retry: 121 delay, repeats = retry 122 assert isinstance(repeats, int) 123 assert repeats > 0 124 self.args = {'mode': mode, 125 'workdir': workdir, 126 'timeout': timeout, 127 'retry': retry, 128 'patch': None, # set during .start 129 } 130 self.alwaysUseLatest = alwaysUseLatest 131 132 # Compute defaults for descriptions: 133 description = ["updating"] 134 descriptionDone = ["update"] 135 if mode == "clobber": 136 description = ["checkout"] 137 # because checkingouting takes too much space 138 descriptionDone = ["checkout"] 139 elif mode == "export": 140 description = ["exporting"] 141 descriptionDone = ["export"] 142 self.description = description 143 self.descriptionDone = descriptionDone

144

145 - def setStepStatus(self, step_status):

146 LoggingBuildStep.setStepStatus(self, step_status)

147

148 - def setDefaultWorkdir(self, workdir):

149 self.args['workdir'] = self.args['workdir'] or workdir

150

151 - def describe(self, done=False):

152 if done: 153 return self.descriptionDone 154 return self.description

155

156 - def computeSourceRevision(self, changes):

157 """Each subclass must implement this method to do something more 158 precise than -rHEAD every time. For version control systems that use 159 repository-wide change numbers (SVN, P4), this can simply take the 160 maximum such number from all the changes involved in this build. For 161 systems that do not (CVS), it needs to create a timestamp based upon 162 the latest Change, the Build's treeStableTimer, and an optional 163 self.checkoutDelay value.""" 164 return None

165

166 - def computeRepositoryURL(self, repository):

167 ''' 168 Helper function that the repository URL based on the parameter the 169 source step took and the Change 'repository' property 170 ''' 171 172 assert not repository or callable(repository) or isinstance(repository, dict) or \ 173 isinstance(repository, str) or isinstance(repository, unicode) 174 175 s = self.build.getSourceStamp() 176 if not repository: 177 assert s.repository 178 return str(s.repository) 179 else: 180 if callable(repository): 181 return str(repository(s.repository)) 182 elif isinstance(repository, dict): 183 return str(repository.get(s.repository)) 184 else: # string or unicode 185 try: 186 repourl = str(repository % s.repository) 187 except TypeError: 188 # that's the backward compatibility case 189 repourl = repository 190 return str(repourl)

191

192 - def start(self):

193 if self.notReally: 194 log.msg("faking %s checkout/update" % self.name) 195 self.step_status.setText(["fake", self.name, "successful"]) 196 self.addCompleteLog("log", 197 "Faked %s checkout/update 'successful'\n" \ 198 % self.name) 199 return SKIPPED 200 201 # what source stamp would this build like to use? 202 s = self.build.getSourceStamp() 203 # if branch is None, then use the Step's "default" branch 204 branch = s.branch or self.branch 205 # if revision is None, use the latest sources (-rHEAD) 206 revision = s.revision 207 if not revision and not self.alwaysUseLatest: 208 revision = self.computeSourceRevision(s.changes) 209 # if patch is None, then do not patch the tree after checkout 210 211 # 'patch' is None or a tuple of (patchlevel, diff, root) 212 # root is optional. 213 patch = s.patch 214 if patch: 215 self.addCompleteLog("patch", patch[1]) 216 217 if self.alwaysUseLatest: 218 revision = None 219 self.startVC(branch, revision, patch)

220

221 - def commandComplete(self, cmd):

222 if cmd.updates.has_key("got_revision"): 223 got_revision = cmd.updates["got_revision"][-1] 224 if got_revision is not None: 225 self.setProperty("got_revision", str(got_revision), "Source")

226 227 228

229 -class BK(Source):

230 """I perform BitKeeper checkout/update operations.""" 231 232 name = 'bk' 233

234 - def __init__(self, bkurl=None, baseURL=None, 235 directory=None, extra_args=None, **kwargs):

236 """ 237 @type bkurl: string 238 @param bkurl: the URL which points to the BitKeeper server. 239 240 @type baseURL: string 241 @param baseURL: if branches are enabled, this is the base URL to 242 which a branch name will be appended. It should 243 probably end in a slash. Use exactly one of 244 C{bkurl} and C{baseURL}. 245 """ 246 247 self.bkurl = bkurl 248 self.baseURL = baseURL 249 self.extra_args = extra_args 250 251 Source.__init__(self, **kwargs) 252 self.addFactoryArguments(bkurl=bkurl, 253 baseURL=baseURL, 254 directory=directory, 255 extra_args=extra_args, 256 ) 257 258 if bkurl and baseURL: 259 raise ValueError("you must use exactly one of bkurl and baseURL")

260 261

262 - def computeSourceRevision(self, changes):

263 return changes.revision

264 265

266 - def startVC(self, branch, revision, patch):

267 268 warnings = [] 269 slavever = self.slaveVersion("bk") 270 if not slavever: 271 m = "slave does not have the 'bk' command" 272 raise BuildSlaveTooOldError(m) 273 274 if self.bkurl: 275 assert not branch # we need baseURL= to use branches 276 self.args['bkurl'] = self.computeRepositoryURL(self.bkurl) 277 else: 278 self.args['bkurl'] = self.computeRepositoryURL(self.baseURL) + branch 279 self.args['revision'] = revision 280 self.args['patch'] = patch 281 self.args['branch'] = branch 282 if self.extra_args is not None: 283 self.args['extra_args'] = self.extra_args 284 285 revstuff = [] 286 revstuff.append("[branch]") 287 if revision is not None: 288 revstuff.append("r%s" % revision) 289 if patch is not None: 290 revstuff.append("[patch]") 291 self.description.extend(revstuff) 292 self.descriptionDone.extend(revstuff) 293 294 cmd = LoggedRemoteCommand("bk", self.args) 295 self.startCommand(cmd, warnings)

296 297 298

299 -class CVS(Source):

300 """I do CVS checkout/update operations. 301 302 Note: if you are doing anonymous/pserver CVS operations, you will need 303 to manually do a 'cvs login' on each buildslave before the slave has any 304 hope of success. XXX: fix then, take a cvs password as an argument and 305 figure out how to do a 'cvs login' on each build 306 """ 307 308 name = "cvs" 309 310 #progressMetrics = ('output',) 311 # 312 # additional things to track: update gives one stderr line per directory 313 # (starting with 'cvs server: Updating ') (and is fairly stable if files 314 # is empty), export gives one line per directory (starting with 'cvs 315 # export: Updating ') and another line per file (starting with U). Would 316 # be nice to track these, requires grepping LogFile data for lines, 317 # parsing each line. Might be handy to have a hook in LogFile that gets 318 # called with each complete line. 319

320 - def __init__(self, cvsroot=None, cvsmodule="", 321 global_options=[], branch=None, checkoutDelay=None, 322 checkout_options=[], export_options=[], extra_options=[], 323 login=None, 324 **kwargs):

325 326 """ 327 @type cvsroot: string 328 @param cvsroot: CVS Repository from which the source tree should 329 be obtained. '/home/warner/Repository' for local 330 or NFS-reachable repositories, 331 ':pserver:anon@foo.com:/cvs' for anonymous CVS, 332 'user@host.com:/cvs' for non-anonymous CVS or 333 CVS over ssh. Lots of possibilities, check the 334 CVS documentation for more. 335 336 @type cvsmodule: string 337 @param cvsmodule: subdirectory of CVS repository that should be 338 retrieved 339 340 @type login: string or None 341 @param login: if not None, a string which will be provided as a 342 password to the 'cvs login' command, used when a 343 :pserver: method is used to access the repository. 344 This login is only needed once, but must be run 345 each time (just before the CVS operation) because 346 there is no way for the buildslave to tell whether 347 it was previously performed or not. 348 349 @type branch: string 350 @param branch: the default branch name, will be used in a '-r' 351 argument to specify which branch of the source tree 352 should be used for this checkout. Defaults to None, 353 which means to use 'HEAD'. 354 355 @type checkoutDelay: int or None 356 @param checkoutDelay: if not None, the number of seconds to put 357 between the last known Change and the 358 timestamp given to the -D argument. This 359 defaults to exactly half of the parent 360 Build's .treeStableTimer, but it could be 361 set to something else if your CVS change 362 notification has particularly weird 363 latency characteristics. 364 365 @type global_options: list of strings 366 @param global_options: these arguments are inserted in the cvs 367 command line, before the 368 'checkout'/'update' command word. See 369 'cvs --help-options' for a list of what 370 may be accepted here. ['-r'] will make 371 the checked out files read only. ['-r', 372 '-R'] will also assume the repository is 373 read-only (I assume this means it won't 374 use locks to insure atomic access to the 375 ,v files). 376 377 @type checkout_options: list of strings 378 @param checkout_options: these arguments are inserted in the cvs 379 command line, after 'checkout' but before 380 branch or revision specifiers. 381 382 @type export_options: list of strings 383 @param export_options: these arguments are inserted in the cvs 384 command line, after 'export' but before 385 branch or revision specifiers. 386 387 @type extra_options: list of strings 388 @param extra_options: these arguments are inserted in the cvs 389 command line, after 'checkout' or 'export' but before 390 branch or revision specifiers. 391 """ 392 393 self.checkoutDelay = checkoutDelay 394 self.branch = branch 395 self.cvsroot = cvsroot 396 397 Source.__init__(self, **kwargs) 398 self.addFactoryArguments(cvsroot=cvsroot, 399 cvsmodule=cvsmodule, 400 global_options=global_options, 401 checkout_options=checkout_options, 402 export_options=export_options, 403 extra_options=extra_options, 404 branch=branch, 405 checkoutDelay=checkoutDelay, 406 login=login, 407 ) 408 409 self.args.update({'cvsmodule': cvsmodule, 410 'global_options': global_options, 411 'checkout_options':checkout_options, 412 'export_options':export_options, 413 'extra_options':extra_options, 414 'login': login, 415 })

416

417 - def computeSourceRevision(self, changes):

418 if not changes: 419 return None 420 lastChange = max([c.when for c in changes]) 421 if self.checkoutDelay is not None: 422 when = lastChange + self.checkoutDelay 423 else: 424 lastSubmit = max([r.submittedAt for r in self.build.requests]) 425 when = (lastChange + lastSubmit) / 2 426 return formatdate(when)

427

428 - def startVC(self, branch, revision, patch):

429 if self.slaveVersionIsOlderThan("cvs", "1.39"): 430 # the slave doesn't know to avoid re-using the same sourcedir 431 # when the branch changes. We have no way of knowing which branch 432 # the last build used, so if we're using a non-default branch and 433 # either 'update' or 'copy' modes, it is safer to refuse to 434 # build, and tell the user they need to upgrade the buildslave. 435 if (branch != self.branch 436 and self.args['mode'] in ("update", "copy")): 437 m = ("This buildslave (%s) does not know about multiple " 438 "branches, and using mode=%s would probably build the " 439 "wrong tree. " 440 "Refusing to build. Please upgrade the buildslave to " 441 "buildbot-0.7.0 or newer." % (self.build.slavename, 442 self.args['mode'])) 443 log.msg(m) 444 raise BuildSlaveTooOldError(m) 445 446 if self.slaveVersionIsOlderThan("cvs", "2.10"): 447 if self.args['extra_options'] or self.args['export_options']: 448 m = ("This buildslave (%s) does not support export_options " 449 "or extra_options arguments to the CVS step." 450 % (self.build.slavename)) 451 log.msg(m) 452 raise BuildSlaveTooOldError(m) 453 # the unwanted args are empty, and will probably be ignored by 454 # the slave, but delete them just to be safe 455 del self.args['export_options'] 456 del self.args['extra_options'] 457 458 if branch is None: 459 branch = "HEAD" 460 self.args['cvsroot'] = self.computeRepositoryURL(self.cvsroot) 461 self.args['branch'] = branch 462 self.args['revision'] = revision 463 self.args['patch'] = patch 464 465 if self.args['branch'] == "HEAD" and self.args['revision']: 466 # special case. 'cvs update -r HEAD -D today' gives no files 467 # TODO: figure out why, see if it applies to -r BRANCH 468 self.args['branch'] = None 469 470 # deal with old slaves 471 warnings = [] 472 slavever = self.slaveVersion("cvs", "old") 473 474 if slavever == "old": 475 # 0.5.0 476 if self.args['mode'] == "export": 477 self.args['export'] = 1 478 elif self.args['mode'] == "clobber": 479 self.args['clobber'] = 1 480 elif self.args['mode'] == "copy": 481 self.args['copydir'] = "source" 482 self.args['tag'] = self.args['branch'] 483 assert not self.args['patch'] # 0.5.0 slave can't do patch 484 485 cmd = LoggedRemoteCommand("cvs", self.args) 486 self.startCommand(cmd, warnings)

487 488

489 -class SVN(Source):

490 """I perform Subversion checkout/update operations.""" 491 492 name = 'svn' 493

494 - def __init__(self, svnurl=None, baseURL=None, defaultBranch=None, 495 directory=None, username=None, password=None, 496 extra_args=None, keep_on_purge=None, ignore_ignores=None, 497 always_purge=None, depth=None, **kwargs):

498 """ 499 @type svnurl: string 500 @param svnurl: the URL which points to the Subversion server, 501 combining the access method (HTTP, ssh, local file), 502 the repository host/port, the repository path, the 503 sub-tree within the repository, and the branch to 504 check out. Use exactly one of C{svnurl} and C{baseURL}. 505 506 @param baseURL: if branches are enabled, this is the base URL to 507 which a branch name will be appended. It should 508 probably end in a slash. Use exactly one of 509 C{svnurl} and C{baseURL}. 510 511 @param defaultBranch: if branches are enabled, this is the branch 512 to use if the Build does not specify one 513 explicitly. It will simply be appended 514 to C{baseURL} and the result handed to 515 the SVN command. 516 517 @type username: string 518 @param username: username to pass to svn's --username 519 520 @type password: string 521 @param password: password to pass to svn's --password 522 """ 523 524 if not 'workdir' in kwargs and directory is not None: 525 # deal with old configs 526 warn("Please use workdir=, not directory=", DeprecationWarning) 527 kwargs['workdir'] = directory 528 529 self.svnurl = svnurl 530 self.baseURL = baseURL 531 self.branch = defaultBranch 532 self.username = username 533 self.password = password 534 self.extra_args = extra_args 535 self.keep_on_purge = keep_on_purge 536 self.ignore_ignores = ignore_ignores 537 self.always_purge = always_purge 538 self.depth = depth 539 540 Source.__init__(self, **kwargs) 541 self.addFactoryArguments(svnurl=svnurl, 542 baseURL=baseURL, 543 defaultBranch=defaultBranch, 544 directory=directory, 545 username=username, 546 password=password, 547 extra_args=extra_args, 548 keep_on_purge=keep_on_purge, 549 ignore_ignores=ignore_ignores, 550 always_purge=always_purge, 551 depth=depth, 552 ) 553 554 if svnurl and baseURL: 555 raise ValueError("you must use either svnurl OR baseURL")

556

557 - def computeSourceRevision(self, changes):

558 if not changes or None in [c.revision for c in changes]: 559 return None 560 lastChange = max([int(c.revision) for c in changes]) 561 return lastChange

562

563 - def startVC(self, branch, revision, patch):

564 565 # handle old slaves 566 warnings = [] 567 slavever = self.slaveVersion("svn", "old") 568 if not slavever: 569 m = "slave does not have the 'svn' command" 570 raise BuildSlaveTooOldError(m) 571 572 if self.slaveVersionIsOlderThan("svn", "1.39"): 573 # the slave doesn't know to avoid re-using the same sourcedir 574 # when the branch changes. We have no way of knowing which branch 575 # the last build used, so if we're using a non-default branch and 576 # either 'update' or 'copy' modes, it is safer to refuse to 577 # build, and tell the user they need to upgrade the buildslave. 578 if (branch != self.branch 579 and self.args['mode'] in ("update", "copy")): 580 m = ("This buildslave (%s) does not know about multiple " 581 "branches, and using mode=%s would probably build the " 582 "wrong tree. " 583 "Refusing to build. Please upgrade the buildslave to " 584 "buildbot-0.7.0 or newer." % (self.build.slavename, 585 self.args['mode'])) 586 raise BuildSlaveTooOldError(m) 587 588 if slavever == "old": 589 # 0.5.0 compatibility 590 if self.args['mode'] in ("clobber", "copy"): 591 # TODO: use some shell commands to make up for the 592 # deficiency, by blowing away the old directory first (thus 593 # forcing a full checkout) 594 warnings.append("WARNING: this slave can only do SVN updates" 595 ", not mode=%s\n" % self.args['mode']) 596 log.msg("WARNING: this slave only does mode=update") 597 if self.args['mode'] == "export": 598 raise BuildSlaveTooOldError("old slave does not have " 599 "mode=export") 600 self.args['directory'] = self.args['workdir'] 601 if revision is not None: 602 # 0.5.0 can only do HEAD. We have no way of knowing whether 603 # the requested revision is HEAD or not, and for 604 # slowly-changing trees this will probably do the right 605 # thing, so let it pass with a warning 606 m = ("WARNING: old slave can only update to HEAD, not " 607 "revision=%s" % revision) 608 log.msg(m) 609 warnings.append(m + "\n") 610 revision = "HEAD" # interprets this key differently 611 if patch: 612 raise BuildSlaveTooOldError("old slave can't do patch") 613 614 if self.svnurl: 615 self.args['svnurl'] = self.computeRepositoryURL(self.svnurl) 616 else: 617 self.args['svnurl'] = (self.computeRepositoryURL(self.baseURL) + 618 branch) 619 self.args['revision'] = revision 620 self.args['patch'] = patch 621 622 self.args['always_purge'] = self.always_purge 623 624 #Set up depth if specified 625 if self.depth is not None: 626 if self.slaveVersionIsOlderThan("svn","2.9"): 627 m = ("This buildslave (%s) does not support svn depth " 628 "arguments. Refusing to build. " 629 "Please upgrade the buildslave." % (self.build.slavename)) 630 raise BuildSlaveTooOldError(m) 631 else: 632 self.args['depth'] = self.depth 633 634 if self.username is not None or self.password is not None: 635 if self.slaveVersionIsOlderThan("svn", "2.8"): 636 m = ("This buildslave (%s) does not support svn usernames " 637 "and passwords. " 638 "Refusing to build. Please upgrade the buildslave to " 639 "buildbot-0.7.10 or newer." % (self.build.slavename,)) 640 raise BuildSlaveTooOldError(m) 641 if self.username is not None: 642 self.args['username'] = self.username 643 if self.password is not None: 644 self.args['password'] = self.password 645 646 if self.extra_args is not None: 647 self.args['extra_args'] = self.extra_args 648 649 revstuff = [] 650 #revstuff.append(self.args['svnurl']) 651 if self.args['svnurl'].find('trunk') == -1: 652 revstuff.append("[branch]") 653 if revision is not None: 654 revstuff.append("r%s" % revision) 655 if patch is not None: 656 revstuff.append("[patch]") 657 self.description.extend(revstuff) 658 self.descriptionDone.extend(revstuff) 659 660 cmd = LoggedRemoteCommand("svn", self.args) 661 self.startCommand(cmd, warnings)

662 663

664 -class Darcs(Source):

665 """Check out a source tree from a Darcs repository at 'repourl'. 666 667 Darcs has no concept of file modes. This means the eXecute-bit will be 668 cleared on all source files. As a result, you may need to invoke 669 configuration scripts with something like: 670 671 C{s(step.Configure, command=['/bin/sh', './configure'])} 672 """ 673 674 name = "darcs" 675

676 - def __init__(self, repourl=None, baseURL=None, defaultBranch=None, 677 **kwargs):

678 """ 679 @type repourl: string 680 @param repourl: the URL which points at the Darcs repository. This 681 is used as the default branch. Using C{repourl} does 682 not enable builds of alternate branches: use 683 C{baseURL} to enable this. Use either C{repourl} or 684 C{baseURL}, not both. 685 686 @param baseURL: if branches are enabled, this is the base URL to 687 which a branch name will be appended. It should 688 probably end in a slash. Use exactly one of 689 C{repourl} and C{baseURL}. 690 691 @param defaultBranch: if branches are enabled, this is the branch 692 to use if the Build does not specify one 693 explicitly. It will simply be appended to 694 C{baseURL} and the result handed to the 695 'darcs pull' command. 696 """ 697 self.repourl = repourl 698 self.baseURL = baseURL 699 self.branch = defaultBranch 700 Source.__init__(self, **kwargs) 701 self.addFactoryArguments(repourl=repourl, 702 baseURL=baseURL, 703 defaultBranch=defaultBranch, 704 ) 705 assert self.args['mode'] != "export", \ 706 "Darcs does not have an 'export' mode" 707 if repourl and baseURL: 708 raise ValueError("you must provide exactly one of repourl and" 709 " baseURL")

710

711 - def startVC(self, branch, revision, patch):

712 slavever = self.slaveVersion("darcs") 713 if not slavever: 714 m = "slave is too old, does not know about darcs" 715 raise BuildSlaveTooOldError(m) 716 717 if self.slaveVersionIsOlderThan("darcs", "1.39"): 718 if revision: 719 # TODO: revisit this once we implement computeSourceRevision 720 m = "0.6.6 slaves can't handle args['revision']" 721 raise BuildSlaveTooOldError(m) 722 723 # the slave doesn't know to avoid re-using the same sourcedir 724 # when the branch changes. We have no way of knowing which branch 725 # the last build used, so if we're using a non-default branch and 726 # either 'update' or 'copy' modes, it is safer to refuse to 727 # build, and tell the user they need to upgrade the buildslave. 728 if (branch != self.branch 729 and self.args['mode'] in ("update", "copy")): 730 m = ("This buildslave (%s) does not know about multiple " 731 "branches, and using mode=%s would probably build the " 732 "wrong tree. " 733 "Refusing to build. Please upgrade the buildslave to " 734 "buildbot-0.7.0 or newer." % (self.build.slavename, 735 self.args['mode'])) 736 raise BuildSlaveTooOldError(m) 737 738 if self.repourl: 739 assert not branch # we need baseURL= to use branches 740 self.args['repourl'] = self.computeRepositoryURL(self.repourl) 741 else: 742 self.args['repourl'] = self.computeRepositoryURL(self.baseURL) + branch 743 self.args['revision'] = revision 744 self.args['patch'] = patch 745 746 revstuff = [] 747 if branch is not None and branch != self.branch: 748 revstuff.append("[branch]") 749 self.description.extend(revstuff) 750 self.descriptionDone.extend(revstuff) 751 752 cmd = LoggedRemoteCommand("darcs", self.args) 753 self.startCommand(cmd)

754 755

756 -class Git(Source):

757 """Check out a source tree from a git repository 'repourl'.""" 758 759 name = "git" 760

761 - def __init__(self, repourl=None, 762 branch="master", 763 submodules=False, 764 ignore_ignores=None, 765 shallow=False, 766 **kwargs):

767 """ 768 @type repourl: string 769 @param repourl: the URL which points at the git repository 770 771 @type branch: string 772 @param branch: The branch or tag to check out by default. If 773 a build specifies a different branch, it will 774 be used instead of this. 775 776 @type submodules: boolean 777 @param submodules: Whether or not to update (and initialize) 778 git submodules. 779 780 @type shallow: boolean 781 @param shallow: Use a shallow or clone, if possible 782 """ 783 Source.__init__(self, **kwargs) 784 self.repourl = repourl 785 self.addFactoryArguments(repourl=repourl, 786 branch=branch, 787 submodules=submodules, 788 ignore_ignores=ignore_ignores, 789 shallow=shallow, 790 ) 791 self.args.update({'branch': branch, 792 'submodules': submodules, 793 'ignore_ignores': ignore_ignores, 794 'shallow': shallow, 795 })

796

797 - def computeSourceRevision(self, changes):

798 if not changes: 799 return None 800 return changes[-1].revision

801

802 - def startVC(self, branch, revision, patch):

803 if branch is not None: 804 self.args['branch'] = branch 805 806 self.args['repourl'] = self.computeRepositoryURL(self.repourl) 807 self.args['revision'] = revision 808 self.args['patch'] = patch 809 slavever = self.slaveVersion("git") 810 if not slavever: 811 raise BuildSlaveTooOldError("slave is too old, does not know " 812 "about git") 813 cmd = LoggedRemoteCommand("git", self.args) 814 self.startCommand(cmd)

815 816

817 -class Arch(Source):

818 """Check out a source tree from an Arch repository named 'archive' 819 available at 'url'. 'version' specifies which version number (development 820 line) will be used for the checkout: this is mostly equivalent to a 821 branch name. This version uses the 'tla' tool to do the checkout, to use 822 'baz' see L{Bazaar} instead. 823 """ 824 825 name = "arch" 826 # TODO: slaves >0.6.6 will accept args['build-config'], so use it 827

828 - def __init__(self, url=None, version=None, archive=None, **kwargs):

829 """ 830 @type url: string 831 @param url: the Arch coordinates of the repository. This is 832 typically an http:// URL, but could also be the absolute 833 pathname of a local directory instead. 834 835 @type version: string 836 @param version: the category--branch--version to check out. This is 837 the default branch. If a build specifies a different 838 branch, it will be used instead of this. 839 840 @type archive: string 841 @param archive: The archive name. If provided, it must match the one 842 that comes from the repository. If not, the 843 repository's default will be used. 844 """ 845 self.branch = version 846 self.url = url 847 Source.__init__(self, **kwargs) 848 self.addFactoryArguments(url=url, 849 version=version, 850 archive=archive, 851 ) 852 assert version, "version should be provided" 853 self.args.update({'archive': archive})

854

855 - def computeSourceRevision(self, changes):

856 # in Arch, fully-qualified revision numbers look like: 857 # arch@buildbot.sourceforge.net--2004/buildbot--dev--0--patch-104 858 # For any given builder, all of this is fixed except the patch-104. 859 # The Change might have any part of the fully-qualified string, so we 860 # just look for the last part. We return the "patch-NN" string. 861 if not changes: 862 return None 863 lastChange = None 864 for c in changes: 865 if not c.revision: 866 continue 867 if c.revision.endswith("--base-0"): 868 rev = 0 869 else: 870 i = c.revision.rindex("patch") 871 rev = int(c.revision[i+len("patch-"):]) 872 lastChange = max(lastChange, rev) 873 if lastChange is None: 874 return None 875 if lastChange == 0: 876 return "base-0" 877 return "patch-%d" % lastChange

878

879 - def checkSlaveVersion(self, cmd, branch):

880 warnings = [] 881 slavever = self.slaveVersion(cmd) 882 if not slavever: 883 m = "slave is too old, does not know about %s" % cmd 884 raise BuildSlaveTooOldError(m) 885 886 # slave 1.28 and later understand 'revision' 887 if self.slaveVersionIsOlderThan(cmd, "1.28"): 888 if not self.alwaysUseLatest: 889 # we don't know whether our requested revision is the latest 890 # or not. If the tree does not change very quickly, this will 891 # probably build the right thing, so emit a warning rather 892 # than refuse to build at all 893 m = "WARNING, buildslave is too old to use a revision" 894 log.msg(m) 895 warnings.append(m + "\n") 896 897 if self.slaveVersionIsOlderThan(cmd, "1.39"): 898 # the slave doesn't know to avoid re-using the same sourcedir 899 # when the branch changes. We have no way of knowing which branch 900 # the last build used, so if we're using a non-default branch and 901 # either 'update' or 'copy' modes, it is safer to refuse to 902 # build, and tell the user they need to upgrade the buildslave. 903 if (branch != self.branch 904 and self.args['mode'] in ("update", "copy")): 905 m = ("This buildslave (%s) does not know about multiple " 906 "branches, and using mode=%s would probably build the " 907 "wrong tree. " 908 "Refusing to build. Please upgrade the buildslave to " 909 "buildbot-0.7.0 or newer." % (self.build.slavename, 910 self.args['mode'])) 911 log.msg(m) 912 raise BuildSlaveTooOldError(m) 913 914 return warnings

915

916 - def startVC(self, branch, revision, patch):

917 self.args['url'] = self.computeRepositoryURL(self.url), 918 self.args['version'] = branch 919 self.args['revision'] = revision 920 self.args['patch'] = patch 921 warnings = self.checkSlaveVersion("arch", branch) 922 923 revstuff = [] 924 if branch is not None and branch != self.branch: 925 revstuff.append("[branch]") 926 if revision is not None: 927 revstuff.append("patch%s" % revision) 928 self.description.extend(revstuff) 929 self.descriptionDone.extend(revstuff) 930 931 cmd = LoggedRemoteCommand("arch", self.args) 932 self.startCommand(cmd, warnings)

933 934

935 -class Bazaar(Arch):

936 """Bazaar is an alternative client for Arch repositories. baz is mostly 937 compatible with tla, but archive registration is slightly different.""" 938 939 # TODO: slaves >0.6.6 will accept args['build-config'], so use it 940

941 - def __init__(self, url, version, archive, **kwargs):

942 """ 943 @type url: string 944 @param url: the Arch coordinates of the repository. This is 945 typically an http:// URL, but could also be the absolute 946 pathname of a local directory instead. 947 948 @type version: string 949 @param version: the category--branch--version to check out 950 951 @type archive: string 952 @param archive: The archive name (required). This must always match 953 the one that comes from the repository, otherwise the 954 buildslave will attempt to get sources from the wrong 955 archive. 956 """ 957 self.branch = version 958 self.url = url 959 Source.__init__(self, **kwargs) 960 self.addFactoryArguments(url=url, 961 version=version, 962 archive=archive, 963 ) 964 self.args.update({'archive': archive, 965 })

966

967 - def startVC(self, branch, revision, patch):

968 self.args['url'] = self.computeRepositoryURL(url), 969 self.args['version'] = branch 970 self.args['revision'] = revision 971 self.args['patch'] = patch 972 warnings = self.checkSlaveVersion("bazaar", branch) 973 974 revstuff = [] 975 if branch is not None and branch != self.branch: 976 revstuff.append("[branch]") 977 if revision is not None: 978 revstuff.append("patch%s" % revision) 979 self.description.extend(revstuff) 980 self.descriptionDone.extend(revstuff) 981 982 cmd = LoggedRemoteCommand("bazaar", self.args) 983 self.startCommand(cmd, warnings)

984

985 -class Bzr(Source):

986 """Check out a source tree from a bzr (Bazaar) repository at 'repourl'. 987 988 """ 989 990 name = "bzr" 991

992 - def __init__(self, repourl=None, baseURL=None, defaultBranch=None, 993 forceSharedRepo=None, 994 **kwargs):

995 """ 996 @type repourl: string 997 @param repourl: the URL which points at the bzr repository. This 998 is used as the default branch. Using C{repourl} does 999 not enable builds of alternate branches: use 1000 C{baseURL} to enable this. Use either C{repourl} or 1001 C{baseURL}, not both. 1002 1003 @param baseURL: if branches are enabled, this is the base URL to 1004 which a branch name will be appended. It should 1005 probably end in a slash. Use exactly one of 1006 C{repourl} and C{baseURL}. 1007 1008 @param defaultBranch: if branches are enabled, this is the branch 1009 to use if the Build does not specify one 1010 explicitly. It will simply be appended to 1011 C{baseURL} and the result handed to the 1012 'bzr checkout pull' command. 1013 1014 1015 @param forceSharedRepo: Boolean, defaults to False. If set to True, 1016 the working directory will be made into a 1017 bzr shared repository if it is not already. 1018 Shared repository greatly reduces the amount 1019 of history data that needs to be downloaded 1020 if not using update/copy mode, or if using 1021 update/copy mode with multiple branches. 1022 """ 1023 self.repourl = repourl 1024 self.baseURL = baseURL 1025 self.branch = defaultBranch 1026 Source.__init__(self, **kwargs) 1027 self.addFactoryArguments(repourl=repourl, 1028 baseURL=baseURL, 1029 defaultBranch=defaultBranch, 1030 forceSharedRepo=forceSharedRepo 1031 ) 1032 self.args.update({'forceSharedRepo': forceSharedRepo}) 1033 if repourl and baseURL: 1034 raise ValueError("you must provide exactly one of repourl and" 1035 " baseURL")

1036

1037 - def computeSourceRevision(self, changes):

1038 if not changes: 1039 return None 1040 lastChange = max([int(c.revision) for c in changes]) 1041 return lastChange

1042

1043 - def startVC(self, branch, revision, patch):

1044 slavever = self.slaveVersion("bzr") 1045 if not slavever: 1046 m = "slave is too old, does not know about bzr" 1047 raise BuildSlaveTooOldError(m) 1048 1049 if self.repourl: 1050 assert not branch # we need baseURL= to use branches 1051 self.args['repourl'] = self.computeRepositoryURL(self.repourl) 1052 else: 1053 self.args['repourl'] = self.computeRepositoryURL(self.baseURL) + branch 1054 self.args['revision'] = revision 1055 self.args['patch'] = patch 1056 1057 revstuff = [] 1058 if branch is not None and branch != self.branch: 1059 revstuff.append("[" + branch + "]") 1060 if revision is not None: 1061 revstuff.append("r%s" % revision) 1062 self.description.extend(revstuff) 1063 self.descriptionDone.extend(revstuff) 1064 1065 cmd = LoggedRemoteCommand("bzr", self.args) 1066 self.startCommand(cmd)

1067 1068

1069 -class Mercurial(Source):

1070 """Check out a source tree from a mercurial repository 'repourl'.""" 1071 1072 name = "hg" 1073

1074 - def __init__(self, repourl=None, baseURL=None, defaultBranch=None, 1075 branchType='dirname', clobberOnBranchChange=True, **kwargs):

1076 """ 1077 @type repourl: string 1078 @param repourl: the URL which points at the Mercurial repository. 1079 This uses the 'default' branch unless defaultBranch is 1080 specified below and the C{branchType} is set to 1081 'inrepo'. It is an error to specify a branch without 1082 setting the C{branchType} to 'inrepo'. 1083 1084 @param baseURL: if 'dirname' branches are enabled, this is the base URL 1085 to which a branch name will be appended. It should 1086 probably end in a slash. Use exactly one of C{repourl} 1087 and C{baseURL}. 1088 1089 @param defaultBranch: if branches are enabled, this is the branch 1090 to use if the Build does not specify one 1091 explicitly. 1092 For 'dirname' branches, It will simply be 1093 appended to C{baseURL} and the result handed to 1094 the 'hg update' command. 1095 For 'inrepo' branches, this specifies the named 1096 revision to which the tree will update after a 1097 clone. 1098 1099 @param branchType: either 'dirname' or 'inrepo' depending on whether 1100 the branch name should be appended to the C{baseURL} 1101 or the branch is a mercurial named branch and can be 1102 found within the C{repourl} 1103 1104 @param clobberOnBranchChange: boolean, defaults to True. If set and 1105 using inrepos branches, clobber the tree 1106 at each branch change. Otherwise, just 1107 update to the branch. 1108 """ 1109 self.repourl = repourl 1110 self.baseURL = baseURL 1111 self.branch = defaultBranch 1112 self.branchType = branchType 1113 self.clobberOnBranchChange = clobberOnBranchChange 1114 Source.__init__(self, **kwargs) 1115 self.addFactoryArguments(repourl=repourl, 1116 baseURL=baseURL, 1117 defaultBranch=defaultBranch, 1118 branchType=branchType, 1119 clobberOnBranchChange=clobberOnBranchChange, 1120 ) 1121 if repourl and baseURL: 1122 raise ValueError("you must provide exactly one of repourl and" 1123 " baseURL")

1124

1125 - def startVC(self, branch, revision, patch):

1126 slavever = self.slaveVersion("hg") 1127 if not slavever: 1128 raise BuildSlaveTooOldError("slave is too old, does not know " 1129 "about hg") 1130 1131 if self.repourl: 1132 # we need baseURL= to use dirname branches 1133 assert self.branchType == 'inrepo' or not branch 1134 self.args['repourl'] = self.computeRepositoryURL(self.repourl) 1135 if branch: 1136 self.args['branch'] = branch 1137 else: 1138 self.args['repourl'] = self.computeRepositoryURL(self.baseURL) + (branch or '') 1139 self.args['revision'] = revision 1140 self.args['patch'] = patch 1141 self.args['clobberOnBranchChange'] = self.clobberOnBranchChange 1142 self.args['branchType'] = self.branchType 1143 1144 revstuff = [] 1145 if branch is not None and branch != self.branch: 1146 revstuff.append("[branch]") 1147 self.description.extend(revstuff) 1148 self.descriptionDone.extend(revstuff) 1149 1150 cmd = LoggedRemoteCommand("hg", self.args) 1151 self.startCommand(cmd)

1152

1153 - def computeSourceRevision(self, changes):

1154 if not changes: 1155 return None 1156 # without knowing the revision ancestry graph, we can't sort the 1157 # changes at all. So for now, assume they were given to us in sorted 1158 # order, and just pay attention to the last one. See ticket #103 for 1159 # more details. 1160 if len(changes) > 1: 1161 log.msg("Mercurial.computeSourceRevision: warning: " 1162 "there are %d changes here, assuming the last one is " 1163 "the most recent" % len(changes)) 1164 return changes[-1].revision

1165 1166

1167 -class P4(Source):

1168 """ P4 is a class for accessing perforce revision control""" 1169 name = "p4" 1170

1171 - def __init__(self, p4base=None, defaultBranch=None, p4port=None, p4user=None, 1172 p4passwd=None, p4extra_views=[], 1173 p4client='buildbot_%(slave)s_%(builder)s', **kwargs):

1174 """ 1175 @type p4base: string 1176 @param p4base: A view into a perforce depot, typically 1177 "//depot/proj/" 1178 1179 @type defaultBranch: string 1180 @param defaultBranch: Identify a branch to build by default. Perforce 1181 is a view based branching system. So, the branch 1182 is normally the name after the base. For example, 1183 branch=1.0 is view=//depot/proj/1.0/... 1184 branch=1.1 is view=//depot/proj/1.1/... 1185 1186 @type p4port: string 1187 @param p4port: Specify the perforce server to connection in the format 1188 <host>:<port>. Example "perforce.example.com:1666" 1189 1190 @type p4user: string 1191 @param p4user: The perforce user to run the command as. 1192 1193 @type p4passwd: string 1194 @param p4passwd: The password for the perforce user. 1195 1196 @type p4extra_views: list of tuples 1197 @param p4extra_views: Extra views to be added to 1198 the client that is being used. 1199 1200 @type p4client: string 1201 @param p4client: The perforce client to use for this buildslave. 1202 """ 1203 1204 self.p4base = p4base 1205 self.branch = defaultBranch 1206 Source.__init__(self, **kwargs) 1207 self.addFactoryArguments(p4base=p4base, 1208 defaultBranch=defaultBranch, 1209 p4port=p4port, 1210 p4user=p4user, 1211 p4passwd=p4passwd, 1212 p4extra_views=p4extra_views, 1213 p4client=p4client, 1214 ) 1215 self.args['p4port'] = p4port 1216 self.args['p4user'] = p4user 1217 self.args['p4passwd'] = p4passwd 1218 self.args['p4extra_views'] = p4extra_views 1219 self.p4client = p4client

1220

1221 - def setBuild(self, build):

1222 Source.setBuild(self, build) 1223 self.args['p4base'] = self.computeRepositoryURL(self.p4base) 1224 self.args['p4client'] = self.p4client % { 1225 'slave': build.slavename, 1226 'builder': build.builder.name, 1227 }

1228

1229 - def computeSourceRevision(self, changes):

1230 if not changes: 1231 return None 1232 lastChange = max([int(c.revision) for c in changes]) 1233 return lastChange

1234

1235 - def startVC(self, branch, revision, patch):

1236 slavever = self.slaveVersion("p4") 1237 assert slavever, "slave is too old, does not know about p4" 1238 args = dict(self.args) 1239 args['branch'] = branch or self.branch 1240 args['revision'] = revision 1241 args['patch'] = patch 1242 cmd = LoggedRemoteCommand("p4", args) 1243 self.startCommand(cmd)

1244

1245 -class P4Sync(Source):

1246 """This is a partial solution for using a P4 source repository. You are 1247 required to manually set up each build slave with a useful P4 1248 environment, which means setting various per-slave environment variables, 1249 and creating a P4 client specification which maps the right files into 1250 the slave's working directory. Once you have done that, this step merely 1251 performs a 'p4 sync' to update that workspace with the newest files. 1252 1253 Each slave needs the following environment: 1254 1255 - PATH: the 'p4' binary must be on the slave's PATH 1256 - P4USER: each slave needs a distinct user account 1257 - P4CLIENT: each slave needs a distinct client specification 1258 1259 You should use 'p4 client' (?) to set up a client view spec which maps 1260 the desired files into $SLAVEBASE/$BUILDERBASE/source . 1261 """ 1262 1263 name = "p4sync" 1264

1265 - def __init__(self, p4port, p4user, p4passwd, p4client, **kwargs):

1266 assert kwargs['mode'] == "copy", "P4Sync can only be used in mode=copy" 1267 self.branch = None 1268 Source.__init__(self, **kwargs) 1269 self.addFactoryArguments(p4port=p4port, 1270 p4user=p4user, 1271 p4passwd=p4passwd, 1272 p4client=p4client, 1273 ) 1274 self.args['p4port'] = p4port 1275 self.args['p4user'] = p4user 1276 self.args['p4passwd'] = p4passwd 1277 self.args['p4client'] = p4client

1278

1279 - def computeSourceRevision(self, changes):

1280 if not changes: 1281 return None 1282 lastChange = max([int(c.revision) for c in changes]) 1283 return lastChange

1284

1285 - def startVC(self, branch, revision, patch):

1286 slavever = self.slaveVersion("p4sync") 1287 assert slavever, "slave is too old, does not know about p4" 1288 cmd = LoggedRemoteCommand("p4sync", self.args) 1289 self.startCommand(cmd)

1290

1291 -class Monotone(Source):

1292 """Check out a revision from a monotone server at 'server_addr', 1293 branch 'branch'. 'revision' specifies which revision id to check 1294 out. 1295 1296 This step will first create a local database, if necessary, and then pull 1297 the contents of the server into the database. Then it will do the 1298 checkout/update from this database.""" 1299 1300 name = "monotone" 1301

1302 - def __init__(self, server_addr=None, branch=None, db_path="monotone.db", 1303 monotone="monotone", 1304 **kwargs):

1305 Source.__init__(self, **kwargs) 1306 self.addFactoryArguments(server_addr=server_addr, 1307 branch=branch, 1308 db_path=db_path, 1309 monotone=monotone, 1310 ) 1311 assert branch, "branch must be specified" 1312 self.args.update({"branch": branch, 1313 "db_path": db_path, 1314 "monotone": monotone})

1315

1316 - def computeSourceRevision(self, changes):

1317 if not changes: 1318 return None 1319 return changes[-1].revision

1320

1321 - def startVC(self):

1322 self.args["server_addr"] = self.computeRepositoryURL(server_addr), 1323 slavever = self.slaveVersion("monotone") 1324 assert slavever, "slave is too old, does not know about monotone" 1325 cmd = LoggedRemoteCommand("monotone", self.args) 1326 self.startCommand(cmd)

1327

Source Code for Module buildbot.steps.source