buildbot.steps.source.oldsource

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 17 from warnings import warn 18 from email.Utils import formatdate 19 from twisted.python import log 20 from twisted.internet import defer 21 from zope.interface import implements 22 from buildbot.process.buildstep import RemoteCommand 23 from buildbot.interfaces import BuildSlaveTooOldError, IRenderable 24 from buildbot.steps.source.base import Source 25

26 -class _ComputeRepositoryURL(object):

27 implements(IRenderable) 28

29 - def __init__(self, step, repository):

30 self.step = step 31 self.repository = repository

32

33 - def getRenderingFor(self, props):

34 ''' 35 Helper function that the repository URL based on the parameter the 36 source step took and the Change 'repository' property 37 ''' 38 39 build = props.getBuild() 40 assert build is not None, "Build should be available *during* a build?" 41 s = build.getSourceStamp(self.step.codebase) 42 43 repository = self.repository 44 45 if not repository: 46 return str(s.repository) 47 else: 48 if callable(repository): 49 d = props.render(repository(s.repository)) 50 elif isinstance(repository, dict): 51 d = props.render(repository.get(s.repository)) 52 elif isinstance(repository, str) or isinstance(repository, unicode): 53 try: 54 return str(repository % s.repository) 55 except TypeError: 56 # that's the backward compatibility case 57 d = props.render(repository) 58 else: 59 d = props.render(repository) 60 61 d.addCallback(str) 62 return d

63

64 -class SlaveSource(Source):

65

66 - def __init__(self, mode='update', retry=None, **kwargs):

67 """ 68 @type mode: string 69 @param mode: the kind of VC operation that is desired: 70 - 'update': specifies that the checkout/update should be 71 performed directly into the workdir. Each build is performed 72 in the same directory, allowing for incremental builds. This 73 minimizes disk space, bandwidth, and CPU time. However, it 74 may encounter problems if the build process does not handle 75 dependencies properly (if you must sometimes do a 'clean 76 build' to make sure everything gets compiled), or if source 77 files are deleted but generated files can influence test 78 behavior (e.g. python's .pyc files), or when source 79 directories are deleted but generated files prevent CVS from 80 removing them. When used with a patched checkout, from a 81 previous buildbot try for instance, it will try to "revert" 82 the changes first and will do a clobber if it is unable to 83 get a clean checkout. The behavior is SCM-dependent. 84 85 - 'copy': specifies that the source-controlled workspace 86 should be maintained in a separate directory (called the 87 'copydir'), using checkout or update as necessary. For each 88 build, a new workdir is created with a copy of the source 89 tree (rm -rf workdir; cp -R -P -p copydir workdir). This 90 doubles the disk space required, but keeps the bandwidth low 91 (update instead of a full checkout). A full 'clean' build 92 is performed each time. This avoids any generated-file 93 build problems, but is still occasionally vulnerable to 94 problems such as a CVS repository being manually rearranged 95 (causing CVS errors on update) which are not an issue with 96 a full checkout. 97 98 - 'clobber': specifies that the working directory should be 99 deleted each time, necessitating a full checkout for each 100 build. This insures a clean build off a complete checkout, 101 avoiding any of the problems described above, but is 102 bandwidth intensive, as the whole source tree must be 103 pulled down for each build. 104 105 - 'export': is like 'clobber', except that e.g. the 'cvs 106 export' command is used to create the working directory. 107 This command removes all VC metadata files (the 108 CVS/.svn/{arch} directories) from the tree, which is 109 sometimes useful for creating source tarballs (to avoid 110 including the metadata in the tar file). Not all VC systems 111 support export. 112 113 @type retry: tuple of ints (delay, repeats) (or None) 114 @param retry: if provided, VC update failures are re-attempted up 115 to REPEATS times, with DELAY seconds between each 116 attempt. Some users have slaves with poor connectivity 117 to their VC repository, and they say that up to 80% of 118 their build failures are due to transient network 119 failures that could be handled by simply retrying a 120 couple times. 121 """ 122 Source.__init__(self, **kwargs) 123 124 assert mode in ("update", "copy", "clobber", "export") 125 if retry: 126 delay, repeats = retry 127 assert isinstance(repeats, int) 128 assert repeats > 0 129 self.args = {'mode': mode, 130 'retry': retry, 131 }

132

133 - def start(self):

134 self.args['workdir'] = self.workdir 135 self.args['logEnviron'] = self.logEnviron 136 self.args['env'] = self.env 137 self.args['timeout'] = self.timeout 138 Source.start(self)

139

140 - def commandComplete(self, cmd):

141 if not cmd.updates.has_key("got_revision"): 142 return 143 got_revision = cmd.updates["got_revision"][-1] 144 if got_revision is None: 145 return 146 147 self.updateSourceProperty('got_revision', str(got_revision))

148 149

150 -class BK(SlaveSource):

151 """I perform BitKeeper checkout/update operations.""" 152 153 name = 'bk' 154 155 renderables = [ 'bkurl', 'baseURL' ] 156

157 - def __init__(self, bkurl=None, baseURL=None, 158 directory=None, extra_args=None, **kwargs):

159 """ 160 @type bkurl: string 161 @param bkurl: the URL which points to the BitKeeper server. 162 163 @type baseURL: string 164 @param baseURL: if branches are enabled, this is the base URL to 165 which a branch name will be appended. It should 166 probably end in a slash. Use exactly one of 167 C{bkurl} and C{baseURL}. 168 """ 169 170 self.bkurl = _ComputeRepositoryURL(bkurl) 171 self.baseURL = _ComputeRepositoryURL(baseURL) 172 self.extra_args = extra_args 173 174 Source.__init__(self, **kwargs) 175 self.addFactoryArguments(bkurl=bkurl, 176 baseURL=baseURL, 177 directory=directory, 178 extra_args=extra_args, 179 ) 180 181 if bkurl and baseURL: 182 raise ValueError("you must use exactly one of bkurl and baseURL")

183 184

185 - def computeSourceRevision(self, changes):

186 return changes.revision

187 188

189 - def startVC(self, branch, revision, patch):

190 191 warnings = [] 192 slavever = self.slaveVersion("bk") 193 if not slavever: 194 m = "slave does not have the 'bk' command" 195 raise BuildSlaveTooOldError(m) 196 197 if self.bkurl: 198 assert not branch # we need baseURL= to use branches 199 self.args['bkurl'] = self.bkurl 200 else: 201 self.args['bkurl'] = self.baseURL + branch 202 self.args['revision'] = revision 203 self.args['patch'] = patch 204 self.args['branch'] = branch 205 if self.extra_args is not None: 206 self.args['extra_args'] = self.extra_args 207 208 revstuff = [] 209 revstuff.append("[branch]") 210 if revision is not None: 211 revstuff.append("r%s" % revision) 212 if patch is not None: 213 revstuff.append("[patch]") 214 self.description.extend(revstuff) 215 self.descriptionDone.extend(revstuff) 216 217 cmd = RemoteCommand("bk", self.args) 218 self.startCommand(cmd, warnings)

219 220 221

222 -class CVS(SlaveSource):

223 """I do CVS checkout/update operations. 224 225 Note: if you are doing anonymous/pserver CVS operations, you will need 226 to manually do a 'cvs login' on each buildslave before the slave has any 227 hope of success. XXX: fix then, take a cvs password as an argument and 228 figure out how to do a 'cvs login' on each build 229 """ 230 231 name = "cvs" 232 233 renderables = [ "cvsroot" ] 234 235 #progressMetrics = ('output',) 236 # 237 # additional things to track: update gives one stderr line per directory 238 # (starting with 'cvs server: Updating ') (and is fairly stable if files 239 # is empty), export gives one line per directory (starting with 'cvs 240 # export: Updating ') and another line per file (starting with U). Would 241 # be nice to track these, requires grepping LogFile data for lines, 242 # parsing each line. Might be handy to have a hook in LogFile that gets 243 # called with each complete line. 244

245 - def __init__(self, cvsroot=None, cvsmodule="", 246 global_options=[], branch=None, checkoutDelay=None, 247 checkout_options=[], export_options=[], extra_options=[], 248 login=None, 249 **kwargs):

250 251 """ 252 @type cvsroot: string 253 @param cvsroot: CVS Repository from which the source tree should 254 be obtained. '/home/warner/Repository' for local 255 or NFS-reachable repositories, 256 ':pserver:anon@foo.com:/cvs' for anonymous CVS, 257 'user@host.com:/cvs' for non-anonymous CVS or 258 CVS over ssh. Lots of possibilities, check the 259 CVS documentation for more. 260 261 @type cvsmodule: string 262 @param cvsmodule: subdirectory of CVS repository that should be 263 retrieved 264 265 @type login: string or None 266 @param login: if not None, a string which will be provided as a 267 password to the 'cvs login' command, used when a 268 :pserver: method is used to access the repository. 269 This login is only needed once, but must be run 270 each time (just before the CVS operation) because 271 there is no way for the buildslave to tell whether 272 it was previously performed or not. 273 274 @type branch: string 275 @param branch: the default branch name, will be used in a '-r' 276 argument to specify which branch of the source tree 277 should be used for this checkout. Defaults to None, 278 which means to use 'HEAD'. 279 280 @type checkoutDelay: int or None 281 @param checkoutDelay: if not None, the number of seconds to put 282 between the last known Change and the 283 timestamp given to the -D argument. This 284 defaults to exactly half of the parent 285 Build's .treeStableTimer, but it could be 286 set to something else if your CVS change 287 notification has particularly weird 288 latency characteristics. 289 290 @type global_options: list of strings 291 @param global_options: these arguments are inserted in the cvs 292 command line, before the 293 'checkout'/'update' command word. See 294 'cvs --help-options' for a list of what 295 may be accepted here. ['-r'] will make 296 the checked out files read only. ['-r', 297 '-R'] will also assume the repository is 298 read-only (I assume this means it won't 299 use locks to insure atomic access to the 300 ,v files). 301 302 @type checkout_options: list of strings 303 @param checkout_options: these arguments are inserted in the cvs 304 command line, after 'checkout' but before 305 branch or revision specifiers. 306 307 @type export_options: list of strings 308 @param export_options: these arguments are inserted in the cvs 309 command line, after 'export' but before 310 branch or revision specifiers. 311 312 @type extra_options: list of strings 313 @param extra_options: these arguments are inserted in the cvs 314 command line, after 'checkout' or 'export' but before 315 branch or revision specifiers. 316 """ 317 318 self.checkoutDelay = checkoutDelay 319 self.branch = branch 320 self.cvsroot = _ComputeRepositoryURL(self, cvsroot) 321 322 SlaveSource.__init__(self, **kwargs) 323 324 self.args.update({'cvsmodule': cvsmodule, 325 'global_options': global_options, 326 'checkout_options':checkout_options, 327 'export_options':export_options, 328 'extra_options':extra_options, 329 'login': login, 330 })

331

332 - def computeSourceRevision(self, changes):

333 if not changes: 334 return None 335 lastChange = max([c.when for c in changes]) 336 if self.checkoutDelay is not None: 337 when = lastChange + self.checkoutDelay 338 else: 339 lastSubmit = max([br.submittedAt for br in self.build.requests]) 340 when = (lastChange + lastSubmit) / 2 341 return formatdate(when)

342

343 - def startVC(self, branch, revision, patch):

344 if self.slaveVersionIsOlderThan("cvs", "1.39"): 345 # the slave doesn't know to avoid re-using the same sourcedir 346 # when the branch changes. We have no way of knowing which branch 347 # the last build used, so if we're using a non-default branch and 348 # either 'update' or 'copy' modes, it is safer to refuse to 349 # build, and tell the user they need to upgrade the buildslave. 350 if (branch != self.branch 351 and self.args['mode'] in ("update", "copy")): 352 m = ("This buildslave (%s) does not know about multiple " 353 "branches, and using mode=%s would probably build the " 354 "wrong tree. " 355 "Refusing to build. Please upgrade the buildslave to " 356 "buildbot-0.7.0 or newer." % (self.build.slavename, 357 self.args['mode'])) 358 log.msg(m) 359 raise BuildSlaveTooOldError(m) 360 361 if self.slaveVersionIsOlderThan("cvs", "2.10"): 362 if self.args['extra_options'] or self.args['export_options']: 363 m = ("This buildslave (%s) does not support export_options " 364 "or extra_options arguments to the CVS step." 365 % (self.build.slavename)) 366 log.msg(m) 367 raise BuildSlaveTooOldError(m) 368 # the unwanted args are empty, and will probably be ignored by 369 # the slave, but delete them just to be safe 370 del self.args['export_options'] 371 del self.args['extra_options'] 372 373 if branch is None: 374 branch = "HEAD" 375 self.args['cvsroot'] = self.cvsroot 376 self.args['branch'] = branch 377 self.args['revision'] = revision 378 self.args['patch'] = patch 379 380 if self.args['branch'] == "HEAD" and self.args['revision']: 381 # special case. 'cvs update -r HEAD -D today' gives no files 382 # TODO: figure out why, see if it applies to -r BRANCH 383 self.args['branch'] = None 384 385 # deal with old slaves 386 warnings = [] 387 slavever = self.slaveVersion("cvs", "old") 388 389 if slavever == "old": 390 # 0.5.0 391 if self.args['mode'] == "export": 392 self.args['export'] = 1 393 elif self.args['mode'] == "clobber": 394 self.args['clobber'] = 1 395 elif self.args['mode'] == "copy": 396 self.args['copydir'] = "source" 397 self.args['tag'] = self.args['branch'] 398 assert not self.args['patch'] # 0.5.0 slave can't do patch 399 400 cmd = RemoteCommand("cvs", self.args) 401 self.startCommand(cmd, warnings)

402 403

404 -class SVN(SlaveSource):

405 """I perform Subversion checkout/update operations.""" 406 407 name = 'svn' 408 branch_placeholder = '%%BRANCH%%' 409 410 renderables = [ 'svnurl', 'baseURL' ] 411

412 - def __init__(self, svnurl=None, baseURL=None, defaultBranch=None, 413 directory=None, username=None, password=None, 414 extra_args=None, keep_on_purge=None, ignore_ignores=None, 415 always_purge=None, depth=None, **kwargs):

416 """ 417 @type svnurl: string 418 @param svnurl: the URL which points to the Subversion server, 419 combining the access method (HTTP, ssh, local file), 420 the repository host/port, the repository path, the 421 sub-tree within the repository, and the branch to 422 check out. Use exactly one of C{svnurl} and C{baseURL}. 423 424 @param baseURL: if branches are enabled, this is the base URL to 425 which a branch name will be appended. It should 426 probably end in a slash. Use exactly one of 427 C{svnurl} and C{baseURL}. 428 429 @param defaultBranch: if branches are enabled, this is the branch 430 to use if the Build does not specify one 431 explicitly. It will simply be appended 432 to C{baseURL} and the result handed to 433 the SVN command. 434 435 @type username: string 436 @param username: username to pass to svn's --username 437 438 @type password: string 439 @param password: password to pass to svn's --password 440 """ 441 442 if not 'workdir' in kwargs and directory is not None: 443 # deal with old configs 444 warn("Please use workdir=, not directory=", DeprecationWarning) 445 kwargs['workdir'] = directory 446 447 self.svnurl = svnurl and _ComputeRepositoryURL(self, svnurl) 448 self.baseURL = _ComputeRepositoryURL(self, baseURL) 449 self.branch = defaultBranch 450 self.username = username 451 self.password = password 452 self.extra_args = extra_args 453 self.keep_on_purge = keep_on_purge 454 self.ignore_ignores = ignore_ignores 455 self.always_purge = always_purge 456 self.depth = depth 457 458 SlaveSource.__init__(self, **kwargs) 459 460 if svnurl and baseURL: 461 raise ValueError("you must use either svnurl OR baseURL")

462

463 - def computeSourceRevision(self, changes):

464 if not changes or None in [c.revision for c in changes]: 465 return None 466 lastChange = max([int(c.revision) for c in changes]) 467 return lastChange

468

469 - def checkCompatibility(self):

470 ''' Handle compatibility between old slaves/svn clients ''' 471 472 slavever = self.slaveVersion("svn", "old") 473 474 if not slavever: 475 m = "slave does not have the 'svn' command" 476 raise BuildSlaveTooOldError(m) 477 478 if self.slaveVersionIsOlderThan("svn", "1.39"): 479 # the slave doesn't know to avoid re-using the same sourcedir 480 # when the branch changes. We have no way of knowing which branch 481 # the last build used, so if we're using a non-default branch and 482 # either 'update' or 'copy' modes, it is safer to refuse to 483 # build, and tell the user they need to upgrade the buildslave. 484 if (self.args['branch'] != self.branch 485 and self.args['mode'] in ("update", "copy")): 486 m = ("This buildslave (%s) does not know about multiple " 487 "branches, and using mode=%s would probably build the " 488 "wrong tree. " 489 "Refusing to build. Please upgrade the buildslave to " 490 "buildbot-0.7.0 or newer." % (self.build.slavename, 491 self.args['mode'])) 492 raise BuildSlaveTooOldError(m) 493 494 if (self.depth is not None) and self.slaveVersionIsOlderThan("svn","2.9"): 495 m = ("This buildslave (%s) does not support svn depth " 496 "arguments. Refusing to build. " 497 "Please upgrade the buildslave." % (self.build.slavename)) 498 raise BuildSlaveTooOldError(m) 499 500 if (self.username is not None or self.password is not None) \ 501 and self.slaveVersionIsOlderThan("svn", "2.8"): 502 m = ("This buildslave (%s) does not support svn usernames " 503 "and passwords. " 504 "Refusing to build. Please upgrade the buildslave to " 505 "buildbot-0.7.10 or newer." % (self.build.slavename,)) 506 raise BuildSlaveTooOldError(m)

507

508 - def getSvnUrl(self, branch):

509 ''' Compute the svn url that will be passed to the svn remote command ''' 510 if self.svnurl: 511 return self.svnurl 512 else: 513 if branch is None: 514 m = ("The SVN source step belonging to builder '%s' does not know " 515 "which branch to work with. This means that the change source " 516 "did not specify a branch and that defaultBranch is None." \ 517 % self.build.builder.name) 518 raise RuntimeError(m) 519 520 computed = self.baseURL 521 522 if self.branch_placeholder in self.baseURL: 523 return computed.replace(self.branch_placeholder, branch) 524 else: 525 return computed + branch

526

527 - def startVC(self, branch, revision, patch):

528 warnings = [] 529 530 self.checkCompatibility() 531 532 self.args['svnurl'] = self.getSvnUrl(branch) 533 self.args['revision'] = revision 534 self.args['patch'] = patch 535 self.args['always_purge'] = self.always_purge 536 537 #Set up depth if specified 538 if self.depth is not None: 539 self.args['depth'] = self.depth 540 541 if self.username is not None: 542 self.args['username'] = self.username 543 if self.password is not None: 544 self.args['password'] = self.password 545 546 if self.extra_args is not None: 547 self.args['extra_args'] = self.extra_args 548 549 revstuff = [] 550 #revstuff.append(self.args['svnurl']) 551 if self.args['svnurl'].find('trunk') == -1: 552 revstuff.append("[branch]") 553 if revision is not None: 554 revstuff.append("r%s" % revision) 555 if patch is not None: 556 revstuff.append("[patch]") 557 self.description.extend(revstuff) 558 self.descriptionDone.extend(revstuff) 559 560 cmd = RemoteCommand("svn", self.args) 561 self.startCommand(cmd, warnings)

562 563

564 -class Darcs(SlaveSource):

565 """Check out a source tree from a Darcs repository at 'repourl'. 566 567 Darcs has no concept of file modes. This means the eXecute-bit will be 568 cleared on all source files. As a result, you may need to invoke 569 configuration scripts with something like: 570 571 C{s(step.Configure, command=['/bin/sh', './configure'])} 572 """ 573 574 name = "darcs" 575 576 renderables = [ 'repourl', 'baseURL' ] 577

578 - def __init__(self, repourl=None, baseURL=None, defaultBranch=None, 579 **kwargs):

580 """ 581 @type repourl: string 582 @param repourl: the URL which points at the Darcs repository. This 583 is used as the default branch. Using C{repourl} does 584 not enable builds of alternate branches: use 585 C{baseURL} to enable this. Use either C{repourl} or 586 C{baseURL}, not both. 587 588 @param baseURL: if branches are enabled, this is the base URL to 589 which a branch name will be appended. It should 590 probably end in a slash. Use exactly one of 591 C{repourl} and C{baseURL}. 592 593 @param defaultBranch: if branches are enabled, this is the branch 594 to use if the Build does not specify one 595 explicitly. It will simply be appended to 596 C{baseURL} and the result handed to the 597 'darcs pull' command. 598 """ 599 self.repourl = _ComputeRepositoryURL(self, repourl) 600 self.baseURL = _ComputeRepositoryURL(self, baseURL) 601 self.branch = defaultBranch 602 SlaveSource.__init__(self, **kwargs) 603 assert self.args['mode'] != "export", \ 604 "Darcs does not have an 'export' mode" 605 if repourl and baseURL: 606 raise ValueError("you must provide exactly one of repourl and" 607 " baseURL")

608

609 - def startVC(self, branch, revision, patch):

610 slavever = self.slaveVersion("darcs") 611 if not slavever: 612 m = "slave is too old, does not know about darcs" 613 raise BuildSlaveTooOldError(m) 614 615 if self.slaveVersionIsOlderThan("darcs", "1.39"): 616 if revision: 617 # TODO: revisit this once we implement computeSourceRevision 618 m = "0.6.6 slaves can't handle args['revision']" 619 raise BuildSlaveTooOldError(m) 620 621 # the slave doesn't know to avoid re-using the same sourcedir 622 # when the branch changes. We have no way of knowing which branch 623 # the last build used, so if we're using a non-default branch and 624 # either 'update' or 'copy' modes, it is safer to refuse to 625 # build, and tell the user they need to upgrade the buildslave. 626 if (branch != self.branch 627 and self.args['mode'] in ("update", "copy")): 628 m = ("This buildslave (%s) does not know about multiple " 629 "branches, and using mode=%s would probably build the " 630 "wrong tree. " 631 "Refusing to build. Please upgrade the buildslave to " 632 "buildbot-0.7.0 or newer." % (self.build.slavename, 633 self.args['mode'])) 634 raise BuildSlaveTooOldError(m) 635 636 if self.repourl: 637 assert not branch # we need baseURL= to use branches 638 self.args['repourl'] = self.repourl 639 else: 640 self.args['repourl'] = self.baseURL + branch 641 self.args['revision'] = revision 642 self.args['patch'] = patch 643 644 revstuff = [] 645 if branch is not None and branch != self.branch: 646 revstuff.append("[branch]") 647 self.description.extend(revstuff) 648 self.descriptionDone.extend(revstuff) 649 650 cmd = RemoteCommand("darcs", self.args) 651 self.startCommand(cmd)

652 653

654 -class Git(SlaveSource):

655 """Check out a source tree from a git repository 'repourl'.""" 656 657 name = "git" 658 659 renderables = [ 'repourl' ] 660

661 - def __init__(self, repourl=None, 662 branch="master", 663 submodules=False, 664 ignore_ignores=None, 665 reference=None, 666 shallow=False, 667 progress=False, 668 **kwargs):

669 """ 670 @type repourl: string 671 @param repourl: the URL which points at the git repository 672 673 @type branch: string 674 @param branch: The branch or tag to check out by default. If 675 a build specifies a different branch, it will 676 be used instead of this. 677 678 @type submodules: boolean 679 @param submodules: Whether or not to update (and initialize) 680 git submodules. 681 682 @type reference: string 683 @param reference: The path to a reference repository to obtain 684 objects from, if any. 685 686 @type shallow: boolean 687 @param shallow: Use a shallow or clone, if possible 688 689 @type progress: boolean 690 @param progress: Pass the --progress option when fetching. This 691 can solve long fetches getting killed due to 692 lack of output, but requires Git 1.7.2+. 693 """ 694 SlaveSource.__init__(self, **kwargs) 695 self.repourl = _ComputeRepositoryURL(self, repourl) 696 self.branch = branch 697 self.args.update({'submodules': submodules, 698 'ignore_ignores': ignore_ignores, 699 'reference': reference, 700 'shallow': shallow, 701 'progress': progress, 702 })

703

704 - def computeSourceRevision(self, changes):

705 if not changes: 706 return None 707 return changes[-1].revision

708

709 - def startVC(self, branch, revision, patch):

710 self.args['branch'] = branch 711 self.args['repourl'] = self.repourl 712 self.args['revision'] = revision 713 self.args['patch'] = patch 714 715 # check if there is any patchset we should fetch from Gerrit 716 if self.build.hasProperty("event.patchSet.ref"): 717 # GerritChangeSource 718 self.args['gerrit_branch'] = self.build.getProperty("event.patchSet.ref") 719 self.updateSourceProperty("gerrit_branch", 720 self.args['gerrit_branch']) 721 else: 722 try: 723 # forced build 724 change = self.build.getProperty("gerrit_change", '').split('/') 725 if len(change) == 2: 726 self.args['gerrit_branch'] = "refs/changes/%2.2d/%d/%d" \ 727 % (int(change[0]) % 100, int(change[0]), int(change[1])) 728 self.updateSourceProperty("gerrit_branch", 729 self.args['gerrit_branch']) 730 except: 731 pass 732 733 slavever = self.slaveVersion("git") 734 if not slavever: 735 raise BuildSlaveTooOldError("slave is too old, does not know " 736 "about git") 737 cmd = RemoteCommand("git", self.args) 738 self.startCommand(cmd)

739 740

741 -class Repo(SlaveSource):

742 """Check out a source tree from a repo repository described by manifest.""" 743 744 name = "repo" 745 746 renderables = [ "manifest_url" ] 747

748 - def __init__(self, 749 manifest_url=None, 750 manifest_branch="master", 751 manifest_file="default.xml", 752 tarball=None, 753 jobs=None, 754 **kwargs):

755 """ 756 @type manifest_url: string 757 @param manifest_url: The URL which points at the repo manifests repository. 758 759 @type manifest_branch: string 760 @param manifest_branch: The manifest branch to check out by default. 761 762 @type manifest_file: string 763 @param manifest_file: The manifest to use for sync. 764 765 """ 766 SlaveSource.__init__(self, **kwargs) 767 self.manifest_url = _ComputeRepositoryURL(self, manifest_url) 768 self.args.update({'manifest_branch': manifest_branch, 769 'manifest_file': manifest_file, 770 'tarball': tarball, 771 'manifest_override_url': None, 772 'jobs': jobs 773 })

774

775 - def computeSourceRevision(self, changes):

776 if not changes: 777 return None 778 return changes[-1].revision

779

780 - def parseDownloadProperty(self, s):

781 """ 782 lets try to be nice in the format we want 783 can support several instances of "repo download proj number/patch" (direct copy paste from gerrit web site) 784 or several instances of "proj number/patch" (simpler version) 785 This feature allows integrator to build with several pending interdependant changes. 786 returns list of repo downloads sent to the buildslave 787 """ 788 import re 789 if s == None: 790 return [] 791 re1 = re.compile("repo download ([^ ]+) ([0-9]+/[0-9]+)") 792 re2 = re.compile("([^ ]+) ([0-9]+/[0-9]+)") 793 re3 = re.compile("([^ ]+)/([0-9]+/[0-9]+)") 794 ret = [] 795 for cur_re in [re1, re2, re3]: 796 res = cur_re.search(s) 797 while res: 798 ret.append("%s %s" % (res.group(1), res.group(2))) 799 s = s[:res.start(0)] + s[res.end(0):] 800 res = cur_re.search(s) 801 return ret

802

803 - def buildDownloadList(self):

804 """taken the changesource and forcebuild property, 805 build the repo download command to send to the slave 806 making this a defereable allow config to tweak this 807 in order to e.g. manage dependancies 808 """ 809 downloads = self.build.getProperty("repo_downloads", []) 810 811 # download patches based on GerritChangeSource events 812 for change in self.build.allChanges(): 813 if (change.properties.has_key("event.type") and 814 change.properties["event.type"] == "patchset-created"): 815 downloads.append("%s %s/%s"% (change.properties["event.change.project"], 816 change.properties["event.change.number"], 817 change.properties["event.patchSet.number"])) 818 819 # download patches based on web site forced build properties: 820 # "repo_d", "repo_d0", .., "repo_d9" 821 # "repo_download", "repo_download0", .., "repo_download9" 822 for propName in ["repo_d"] + ["repo_d%d" % i for i in xrange(0,10)] + \ 823 ["repo_download"] + ["repo_download%d" % i for i in xrange(0,10)]: 824 s = self.build.getProperty(propName) 825 if s is not None: 826 downloads.extend(self.parseDownloadProperty(s)) 827 828 if downloads: 829 self.args["repo_downloads"] = downloads 830 self.updateSourceProperty("repo_downloads", downloads) 831 return defer.succeed(None)

832

833 - def startVC(self, branch, revision, patch):

834 self.args['manifest_url'] = self.manifest_url 835 836 # manifest override 837 self.args['manifest_override_url'] = None 838 try: 839 self.args['manifest_override_url'] = self.build.getProperty("manifest_override_url") 840 except KeyError: 841 pass 842 # only master has access to properties, so we must implement this here. 843 d = self.buildDownloadList() 844 d.addCallback(self.continueStartVC, branch, revision, patch) 845 d.addErrback(self.failed)

846

847 - def continueStartVC(self, ignored, branch, revision, patch):

848 slavever = self.slaveVersion("repo") 849 if not slavever: 850 raise BuildSlaveTooOldError("slave is too old, does not know " 851 "about repo") 852 cmd = RemoteCommand("repo", self.args) 853 self.startCommand(cmd)

854

855 - def commandComplete(self, cmd):

856 repo_downloaded = [] 857 if cmd.updates.has_key("repo_downloaded"): 858 repo_downloaded = cmd.updates["repo_downloaded"][-1] 859 if repo_downloaded: 860 self.updateSourceProperty("repo_downloaded", 861 str(repo_downloaded)) 862 else: 863 repo_downloaded = [] 864 orig_downloads = self.getProperty("repo_downloads") or [] 865 if len(orig_downloads) != len(repo_downloaded): 866 self.step_status.setText(["repo download issues"])

867 868

869 -class Bzr(SlaveSource):

870 """Check out a source tree from a bzr (Bazaar) repository at 'repourl'. 871 872 """ 873 874 name = "bzr" 875 876 renderables = [ 'repourl', 'baseURL' ] 877

878 - def __init__(self, repourl=None, baseURL=None, defaultBranch=None, 879 forceSharedRepo=None, 880 **kwargs):

881 """ 882 @type repourl: string 883 @param repourl: the URL which points at the bzr repository. This 884 is used as the default branch. Using C{repourl} does 885 not enable builds of alternate branches: use 886 C{baseURL} to enable this. Use either C{repourl} or 887 C{baseURL}, not both. 888 889 @param baseURL: if branches are enabled, this is the base URL to 890 which a branch name will be appended. It should 891 probably end in a slash. Use exactly one of 892 C{repourl} and C{baseURL}. 893 894 @param defaultBranch: if branches are enabled, this is the branch 895 to use if the Build does not specify one 896 explicitly. It will simply be appended to 897 C{baseURL} and the result handed to the 898 'bzr checkout pull' command. 899 900 901 @param forceSharedRepo: Boolean, defaults to False. If set to True, 902 the working directory will be made into a 903 bzr shared repository if it is not already. 904 Shared repository greatly reduces the amount 905 of history data that needs to be downloaded 906 if not using update/copy mode, or if using 907 update/copy mode with multiple branches. 908 """ 909 self.repourl = _ComputeRepositoryURL(self, repourl) 910 self.baseURL = _ComputeRepositoryURL(self, baseURL) 911 self.branch = defaultBranch 912 SlaveSource.__init__(self, **kwargs) 913 self.args.update({'forceSharedRepo': forceSharedRepo}) 914 if repourl and baseURL: 915 raise ValueError("you must provide exactly one of repourl and" 916 " baseURL")

917

918 - def computeSourceRevision(self, changes):

919 if not changes: 920 return None 921 lastChange = max([int(c.revision) for c in changes]) 922 return lastChange

923

924 - def startVC(self, branch, revision, patch):

925 slavever = self.slaveVersion("bzr") 926 if not slavever: 927 m = "slave is too old, does not know about bzr" 928 raise BuildSlaveTooOldError(m) 929 930 if self.repourl: 931 assert not branch # we need baseURL= to use branches 932 self.args['repourl'] = self.repourl 933 else: 934 self.args['repourl'] = self.baseURL + branch 935 self.args['revision'] = revision 936 self.args['patch'] = patch 937 938 revstuff = [] 939 if branch is not None and branch != self.branch: 940 revstuff.append("[" + branch + "]") 941 if revision is not None: 942 revstuff.append("r%s" % revision) 943 self.description.extend(revstuff) 944 self.descriptionDone.extend(revstuff) 945 946 cmd = RemoteCommand("bzr", self.args) 947 self.startCommand(cmd)

948 949

950 -class Mercurial(SlaveSource):

951 """Check out a source tree from a mercurial repository 'repourl'.""" 952 953 name = "hg" 954 955 renderables = [ 'repourl', 'baseURL' ] 956

957 - def __init__(self, repourl=None, baseURL=None, defaultBranch=None, 958 branchType='dirname', clobberOnBranchChange=True, **kwargs):

959 """ 960 @type repourl: string 961 @param repourl: the URL which points at the Mercurial repository. 962 This uses the 'default' branch unless defaultBranch is 963 specified below and the C{branchType} is set to 964 'inrepo'. It is an error to specify a branch without 965 setting the C{branchType} to 'inrepo'. 966 967 @param baseURL: if 'dirname' branches are enabled, this is the base URL 968 to which a branch name will be appended. It should 969 probably end in a slash. Use exactly one of C{repourl} 970 and C{baseURL}. 971 972 @param defaultBranch: if branches are enabled, this is the branch 973 to use if the Build does not specify one 974 explicitly. 975 For 'dirname' branches, It will simply be 976 appended to C{baseURL} and the result handed to 977 the 'hg update' command. 978 For 'inrepo' branches, this specifies the named 979 revision to which the tree will update after a 980 clone. 981 982 @param branchType: either 'dirname' or 'inrepo' depending on whether 983 the branch name should be appended to the C{baseURL} 984 or the branch is a mercurial named branch and can be 985 found within the C{repourl} 986 987 @param clobberOnBranchChange: boolean, defaults to True. If set and 988 using inrepos branches, clobber the tree 989 at each branch change. Otherwise, just 990 update to the branch. 991 """ 992 self.repourl = _ComputeRepositoryURL(self, repourl) 993 self.baseURL = _ComputeRepositoryURL(self, baseURL) 994 self.branch = defaultBranch 995 self.branchType = branchType 996 self.clobberOnBranchChange = clobberOnBranchChange 997 SlaveSource.__init__(self, **kwargs) 998 if repourl and baseURL: 999 raise ValueError("you must provide exactly one of repourl and" 1000 " baseURL")

1001

1002 - def startVC(self, branch, revision, patch):

1003 slavever = self.slaveVersion("hg") 1004 if not slavever: 1005 raise BuildSlaveTooOldError("slave is too old, does not know " 1006 "about hg") 1007 1008 if self.repourl: 1009 # we need baseURL= to use dirname branches 1010 assert self.branchType == 'inrepo' or not branch 1011 self.args['repourl'] = self.repourl 1012 if branch: 1013 self.args['branch'] = branch 1014 else: 1015 self.args['repourl'] = self.baseURL + (branch or '') 1016 self.args['revision'] = revision 1017 self.args['patch'] = patch 1018 self.args['clobberOnBranchChange'] = self.clobberOnBranchChange 1019 self.args['branchType'] = self.branchType 1020 1021 revstuff = [] 1022 if branch is not None and branch != self.branch: 1023 revstuff.append("[branch]") 1024 self.description.extend(revstuff) 1025 self.descriptionDone.extend(revstuff) 1026 1027 cmd = RemoteCommand("hg", self.args) 1028 self.startCommand(cmd)

1029

1030 - def computeSourceRevision(self, changes):

1031 if not changes: 1032 return None 1033 # without knowing the revision ancestry graph, we can't sort the 1034 # changes at all. So for now, assume they were given to us in sorted 1035 # order, and just pay attention to the last one. See ticket #103 for 1036 # more details. 1037 if len(changes) > 1: 1038 log.msg("Mercurial.computeSourceRevision: warning: " 1039 "there are %d changes here, assuming the last one is " 1040 "the most recent" % len(changes)) 1041 return changes[-1].revision

1042 1043

1044 -class P4(SlaveSource):

1045 """ P4 is a class for accessing perforce revision control""" 1046 name = "p4" 1047 1048 renderables = [ 'p4base' ] 1049

1050 - def __init__(self, p4base=None, defaultBranch=None, p4port=None, p4user=None, 1051 p4passwd=None, p4extra_views=[], p4line_end='local', 1052 p4client='buildbot_%(slave)s_%(builder)s', **kwargs):

1053 """ 1054 @type p4base: string 1055 @param p4base: A view into a perforce depot, typically 1056 "//depot/proj/" 1057 1058 @type defaultBranch: string 1059 @param defaultBranch: Identify a branch to build by default. Perforce 1060 is a view based branching system. So, the branch 1061 is normally the name after the base. For example, 1062 branch=1.0 is view=//depot/proj/1.0/... 1063 branch=1.1 is view=//depot/proj/1.1/... 1064 1065 @type p4port: string 1066 @param p4port: Specify the perforce server to connection in the format 1067 <host>:<port>. Example "perforce.example.com:1666" 1068 1069 @type p4user: string 1070 @param p4user: The perforce user to run the command as. 1071 1072 @type p4passwd: string 1073 @param p4passwd: The password for the perforce user. 1074 1075 @type p4extra_views: list of tuples 1076 @param p4extra_views: Extra views to be added to 1077 the client that is being used. 1078 1079 @type p4line_end: string 1080 @param p4line_end: value of the LineEnd client specification property 1081 1082 @type p4client: string 1083 @param p4client: The perforce client to use for this buildslave. 1084 """ 1085 1086 self.p4base = _ComputeRepositoryURL(self, p4base) 1087 self.branch = defaultBranch 1088 SlaveSource.__init__(self, **kwargs) 1089 self.args['p4port'] = p4port 1090 self.args['p4user'] = p4user 1091 self.args['p4passwd'] = p4passwd 1092 self.args['p4extra_views'] = p4extra_views 1093 self.args['p4line_end'] = p4line_end 1094 self.p4client = p4client

1095

1096 - def setBuild(self, build):

1097 SlaveSource.setBuild(self, build) 1098 self.args['p4client'] = self.p4client % { 1099 'slave': build.slavename, 1100 'builder': build.builder.name, 1101 }

1102

1103 - def computeSourceRevision(self, changes):

1104 if not changes: 1105 return None 1106 lastChange = max([int(c.revision) for c in changes]) 1107 return lastChange

1108

1109 - def startVC(self, branch, revision, patch):

1110 slavever = self.slaveVersion("p4") 1111 assert slavever, "slave is too old, does not know about p4" 1112 args = dict(self.args) 1113 args['p4base'] = self.p4base 1114 args['branch'] = branch or self.branch 1115 args['revision'] = revision 1116 args['patch'] = patch 1117 cmd = RemoteCommand("p4", args) 1118 self.startCommand(cmd)

1119

1120 -class Monotone(SlaveSource):

1121 """Check out a source tree from a monotone repository 'repourl'.""" 1122 1123 name = "mtn" 1124 1125 renderables = [ 'repourl' ] 1126

1127 - def __init__(self, repourl=None, branch=None, progress=False, **kwargs):

1128 """ 1129 @type repourl: string 1130 @param repourl: the URI which points at the monotone repository. 1131 1132 @type branch: string 1133 @param branch: The branch or tag to check out by default. If 1134 a build specifies a different branch, it will 1135 be used instead of this. 1136 1137 @type progress: boolean 1138 @param progress: Pass the --ticker=dot option when pulling. This 1139 can solve long fetches getting killed due to 1140 lack of output. 1141 """ 1142 SlaveSource.__init__(self, **kwargs) 1143 self.repourl = _ComputeRepositoryURL(self, repourl) 1144 if (not repourl): 1145 raise ValueError("you must provide a repository uri in 'repourl'") 1146 if (not branch): 1147 raise ValueError("you must provide a default branch in 'branch'") 1148 self.args.update({'branch': branch, 1149 'progress': progress, 1150 })

1151

1152 - def startVC(self, branch, revision, patch):

1153 slavever = self.slaveVersion("mtn") 1154 if not slavever: 1155 raise BuildSlaveTooOldError("slave is too old, does not know " 1156 "about mtn") 1157 1158 self.args['repourl'] = self.repourl 1159 if branch: 1160 self.args['branch'] = branch 1161 self.args['revision'] = revision 1162 self.args['patch'] = patch 1163 1164 cmd = RemoteCommand("mtn", self.args) 1165 self.startCommand(cmd)

1166

1167 - def computeSourceRevision(self, changes):

1168 if not changes: 1169 return None 1170 # without knowing the revision ancestry graph, we can't sort the 1171 # changes at all. So for now, assume they were given to us in sorted 1172 # order, and just pay attention to the last one. See ticket #103 for 1173 # more details. 1174 if len(changes) > 1: 1175 log.msg("Monotone.computeSourceRevision: warning: " 1176 "there are %d changes here, assuming the last one is " 1177 "the most recent" % len(changes)) 1178 return changes[-1].revision

1179

Source Code for Module buildbot.steps.source.oldsource