buildbot.scripts.runner

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 17 # N.B.: don't import anything that might pull in a reactor yet. Some of our 18 # subcommands want to load modules that need the gtk reactor. 19 # 20 # Also don't forget to mirror your changes on command-line options in manual 21 # pages and texinfo documentation. 22 23 import copy 24 import os, sys, stat, re, time 25 from twisted.python import usage, util, runtime 26 from twisted.internet import defer 27 28 from buildbot.interfaces import BuildbotNotRunningError

29 30 -def in_reactor(f):

31 """decorate a function by running it with maybeDeferred in a reactor""" 32 def wrap(*args, **kwargs): 33 from twisted.internet import reactor 34 result = [ ] 35 def async(): 36 d = defer.maybeDeferred(f, *args, **kwargs) 37 def eb(f): 38 f.printTraceback()

39 d.addErrback(eb) 40 def do_stop(r): 41 result.append(r) 42 reactor.stop() 43 d.addBoth(do_stop) 44 reactor.callWhenRunning(async) 45 reactor.run() 46 return result[0] 47 wrap.__doc__ = f.__doc__ 48 wrap.__name__ = f.__name__ 49 wrap._orig = f # for tests 50 return wrap 51

52 -def isBuildmasterDir(dir):

53 buildbot_tac = os.path.join(dir, "buildbot.tac") 54 if not os.path.isfile(buildbot_tac): 55 print "no buildbot.tac" 56 return False 57 58 contents = open(buildbot_tac, "r").read() 59 return "Application('buildmaster')" in contents

60

61 # the create/start/stop commands should all be run as the same user, 62 # preferably a separate 'buildbot' account. 63 64 # Note that the terms 'options' and 'config' are used intechangeably here - in 65 # fact, they are intercanged several times. Caveat legator. 66 67 -class OptionsWithOptionsFile(usage.Options):

68 # subclasses should set this to a list-of-lists in order to source the 69 # .buildbot/options file. 70 # buildbotOptions = [ [ 'optfile-name', 'option-name' ], .. ] 71 buildbotOptions = None 72

73 - def __init__(self, *args):

74 # for options in self.buildbotOptions, optParameters, and the options 75 # file, change the default in optParameters *before* calling through 76 # to the parent constructor 77 78 # Options uses reflect.accumulateClassList, so this *must* be 79 # a class attribute; however, we do not want to permanently change 80 # the class. So we patch it temporarily and restore it after. 81 cls = self.__class__ 82 if hasattr(cls, 'optParameters'): 83 old_optParameters = cls.optParameters 84 cls.optParameters = op = copy.deepcopy(cls.optParameters) 85 if self.buildbotOptions: 86 optfile = loadOptionsFile() 87 for optfile_name, option_name in self.buildbotOptions: 88 for i in range(len(op)): 89 if (op[i][0] == option_name and optfile_name in optfile): 90 op[i] = list(op[i]) 91 op[i][2] = optfile[optfile_name] 92 usage.Options.__init__(self, *args) 93 if hasattr(cls, 'optParameters'): 94 cls.optParameters = old_optParameters

95

96 -def loadOptionsFile():

97 """Find the .buildbot/FILENAME file. Crawl from the current directory up 98 towards the root, and also look in ~/.buildbot . The first directory 99 that's owned by the user and has the file we're looking for wins. Windows 100 skips the owned-by-user test. 101 102 @rtype: dict 103 @return: a dictionary of names defined in the options file. If no options 104 file was found, return an empty dict. 105 """ 106 107 here = os.path.abspath(os.getcwd()) 108 109 if runtime.platformType == 'win32': 110 # never trust env-vars, use the proper API 111 from win32com.shell import shellcon, shell 112 appdata = shell.SHGetFolderPath(0, shellcon.CSIDL_APPDATA, 0, 0) 113 home = os.path.join(appdata, "buildbot") 114 else: 115 home = os.path.expanduser("~/.buildbot") 116 117 searchpath = [] 118 toomany = 20 119 while True: 120 searchpath.append(os.path.join(here, ".buildbot")) 121 next = os.path.dirname(here) 122 if next == here: 123 break # we've hit the root 124 here = next 125 toomany -= 1 # just in case 126 if toomany == 0: 127 raise ValueError("Hey, I seem to have wandered up into the " 128 "infinite glories of the heavens. Oops.") 129 searchpath.append(home) 130 131 localDict = {} 132 133 for d in searchpath: 134 if os.path.isdir(d): 135 if runtime.platformType != 'win32': 136 if os.stat(d)[stat.ST_UID] != os.getuid(): 137 print "skipping %s because you don't own it" % d 138 continue # security, skip other people's directories 139 optfile = os.path.join(d, "options") 140 if os.path.exists(optfile): 141 try: 142 f = open(optfile, "r") 143 options = f.read() 144 exec options in localDict 145 except: 146 print "error while reading %s" % optfile 147 raise 148 break 149 150 for k in localDict.keys(): 151 if k.startswith("__"): 152 del localDict[k] 153 return localDict

154

155 -class MakerBase(OptionsWithOptionsFile):

156 optFlags = [ 157 ['help', 'h', "Display this message"], 158 ["quiet", "q", "Do not emit the commands being run"], 159 ] 160 161 usage.Options.longdesc = """ 162 Operates upon the specified <basedir> (or the current directory, if not 163 specified). 164 """ 165 166 opt_h = usage.Options.opt_help 167

168 - def parseArgs(self, *args):

169 if len(args) > 0: 170 self['basedir'] = args[0] 171 else: 172 # Use the current directory if no basedir was specified. 173 self['basedir'] = os.getcwd() 174 if len(args) > 1: 175 raise usage.UsageError("I wasn't expecting so many arguments")

176

177 - def postOptions(self):

178 self['basedir'] = os.path.abspath(self['basedir'])

179 180 makefile_sample = """# -*- makefile -*- 181 182 # This is a simple makefile which lives in a buildmaster 183 # directory (next to the buildbot.tac file). It allows you to start/stop the 184 # master by doing 'make start' or 'make stop'. 185 186 # The 'reconfig' target will tell a buildmaster to reload its config file. 187 188 start: 189 twistd --no_save -y buildbot.tac 190 191 stop: 192 if [ -e twistd.pid ]; \\ 193 then kill `cat twistd.pid`; \\ 194 else echo "Nothing to stop."; \\ 195 fi 196 197 reconfig: 198 if [ -e twistd.pid ]; \\ 199 then kill -HUP `cat twistd.pid`; \\ 200 else echo "Nothing to reconfig."; \\ 201 fi 202 203 log: 204 if [ -e twistd.log ]; \\ 205 then tail -f twistd.log; \\ 206 else echo "Nothing to tail."; \\ 207 fi 208 """

209 210 -class Maker:

211 - def __init__(self, config):

212 self.config = config 213 self.basedir = config['basedir'] 214 self.force = config.get('force', False) 215 self.quiet = config['quiet']

216

217 - def mkdir(self):

218 if os.path.exists(self.basedir): 219 if not self.quiet: 220 print "updating existing installation" 221 return 222 if not self.quiet: print "mkdir", self.basedir 223 os.mkdir(self.basedir)

224

225 - def chdir(self):

226 if not self.quiet: print "chdir", self.basedir 227 os.chdir(self.basedir)

228

229 - def makeTAC(self, contents, secret=False):

230 tacfile = "buildbot.tac" 231 if os.path.exists(tacfile): 232 oldcontents = open(tacfile, "rt").read() 233 if oldcontents == contents: 234 if not self.quiet: 235 print "buildbot.tac already exists and is correct" 236 return 237 if not self.quiet: 238 print "not touching existing buildbot.tac" 239 print "creating buildbot.tac.new instead" 240 tacfile = "buildbot.tac.new" 241 f = open(tacfile, "wt") 242 f.write(contents) 243 f.close() 244 if secret: 245 os.chmod(tacfile, 0600)

246

247 - def makefile(self):

248 target = "Makefile.sample" 249 if os.path.exists(target): 250 oldcontents = open(target, "rt").read() 251 if oldcontents == makefile_sample: 252 if not self.quiet: 253 print "Makefile.sample already exists and is correct" 254 return 255 if not self.quiet: 256 print "replacing Makefile.sample" 257 else: 258 if not self.quiet: 259 print "creating Makefile.sample" 260 f = open(target, "wt") 261 f.write(makefile_sample) 262 f.close()

263

264 - def sampleconfig(self, source):

265 target = "master.cfg.sample" 266 config_sample = open(source, "rt").read() 267 if os.path.exists(target): 268 oldcontents = open(target, "rt").read() 269 if oldcontents == config_sample: 270 if not self.quiet: 271 print "master.cfg.sample already exists and is up-to-date" 272 return 273 if not self.quiet: 274 print "replacing master.cfg.sample" 275 else: 276 if not self.quiet: 277 print "creating master.cfg.sample" 278 f = open(target, "wt") 279 f.write(config_sample) 280 f.close() 281 os.chmod(target, 0600)

282

283 - def public_html(self, files):

284 webdir = os.path.join(self.basedir, "public_html") 285 if os.path.exists(webdir): 286 if not self.quiet: 287 print "public_html/ already exists: not replacing" 288 return 289 else: 290 os.mkdir(webdir) 291 if not self.quiet: 292 print "populating public_html/" 293 for target, source in files.iteritems(): 294 target = os.path.join(webdir, target) 295 f = open(target, "wt") 296 f.write(open(source, "rt").read()) 297 f.close()

298

299 - def create_db(self):

300 from buildbot.db import connector 301 from buildbot.master import BuildMaster 302 db = connector.DBConnector(BuildMaster(self.basedir), 303 self.config['db'], basedir=self.basedir) 304 if not self.config['quiet']: print "creating database" 305 d = db.model.upgrade() 306 return d

307

308 - def populate_if_missing(self, target, source, overwrite=False):

309 new_contents = open(source, "rt").read() 310 if os.path.exists(target): 311 old_contents = open(target, "rt").read() 312 if old_contents != new_contents: 313 if overwrite: 314 if not self.quiet: 315 print "%s has old/modified contents" % target 316 print " overwriting it with new contents" 317 open(target, "wt").write(new_contents) 318 else: 319 if not self.quiet: 320 print "%s has old/modified contents" % target 321 print " writing new contents to %s.new" % target 322 open(target + ".new", "wt").write(new_contents) 323 # otherwise, it's up to date 324 else: 325 if not self.quiet: 326 print "populating %s" % target 327 open(target, "wt").write(new_contents)

328

329 - def move_if_present(self, source, dest):

330 if os.path.exists(source): 331 if os.path.exists(dest): 332 print "Notice: %s now overrides %s" % (dest, source) 333 print " as the latter is not used by buildbot anymore." 334 print " Decide which one you want to keep." 335 else: 336 try: 337 print "Notice: Moving %s to %s." % (source, dest) 338 print " You can (and probably want to) remove it if you haven't modified this file." 339 os.renames(source, dest) 340 except Exception, e: 341 print "Error moving %s to %s: %s" % (source, dest, str(e))

342

343 - def upgrade_public_html(self, files):

344 webdir = os.path.join(self.basedir, "public_html") 345 if not os.path.exists(webdir): 346 if not self.quiet: 347 print "populating public_html/" 348 os.mkdir(webdir) 349 for target, source in files.iteritems(): 350 self.populate_if_missing(os.path.join(webdir, target), 351 source)

352

353 - def check_master_cfg(self, expected_db_url=None):

354 """Check the buildmaster configuration, returning a deferred that 355 fires with an approprate exit status (so 0=success).""" 356 from buildbot.master import BuildMaster 357 from twisted.python import log 358 359 master_cfg = os.path.join(self.basedir, "master.cfg") 360 if not os.path.exists(master_cfg): 361 if not self.quiet: 362 print "No master.cfg found" 363 return defer.succeed(1) 364 365 # side-effects of loading the config file: 366 367 # for each Builder defined in c['builders'], if the status directory 368 # didn't already exist, it will be created, and the 369 # $BUILDERNAME/builder pickle might be created (with a single 370 # "builder created" event). 371 372 # we put basedir in front of sys.path, because that's how the 373 # buildmaster itself will run, and it is quite common to have the 374 # buildmaster import helper classes from other .py files in its 375 # basedir. 376 377 if sys.path[0] != self.basedir: 378 sys.path.insert(0, self.basedir) 379 380 m = BuildMaster(self.basedir) 381 382 # we need to route log.msg to stdout, so any problems can be seen 383 # there. But if everything goes well, I'd rather not clutter stdout 384 # with log messages. So instead we add a logObserver which gathers 385 # messages and only displays them if something goes wrong. 386 messages = [] 387 log.addObserver(messages.append) 388 389 # this will errback if there's something wrong with the config file. 390 # Note that this BuildMaster instance is never started, so it won't 391 # actually do anything with the configuration. 392 d = defer.maybeDeferred(lambda : 393 m.loadConfig(open(master_cfg, "r"), checkOnly=True)) 394 def check_db_url(config): 395 if (expected_db_url and 396 config.get('db_url', 'sqlite:///state.sqlite') != expected_db_url): 397 raise ValueError("c['db_url'] in the config file ('%s') does" 398 " not match '%s'; please edit the configuration" 399 " file before upgrading." % 400 (config['db_url'], expected_db_url))

401 d.addCallback(check_db_url) 402 def cb(_): 403 return 0

404 def eb(f): 405 if not self.quiet: 406 print 407 for m in messages: 408 print "".join(m['message']) 409 f.printTraceback() 410 print 411 print "An error was detected in the master.cfg file." 412 print "Please correct the problem and run 'buildbot upgrade-master' again." 413 print 414 return 1 415 d.addCallbacks(cb, eb) 416 return d 417 418 DB_HELP = """ 419 The --db string is evaluated to build the DB object, which specifies 420 which database the buildmaster should use to hold scheduler state and 421 status information. The default (which creates an SQLite database in 422 BASEDIR/state.sqlite) is equivalent to: 423 424 --db='sqlite:///state.sqlite' 425 426 To use a remote MySQL database instead, use something like: 427 428 --db='mysql://bbuser:bbpasswd@dbhost/bbdb' 429 """

430 431 -class UpgradeMasterOptions(MakerBase):

432 optFlags = [ 433 ["replace", "r", "Replace any modified files without confirmation."], 434 ] 435 optParameters = [ 436 ["db", None, "sqlite:///state.sqlite", 437 "which DB to use for scheduler/status state. See below for syntax."], 438 ] 439

440 - def getSynopsis(self):

441 return "Usage: buildbot upgrade-master [options] [<basedir>]"

442 443 longdesc = """ 444 This command takes an existing buildmaster working directory and 445 adds/modifies the files there to work with the current version of 446 buildbot. When this command is finished, the buildmaster directory should 447 look much like a brand-new one created by the 'create-master' command. 448 449 Use this after you've upgraded your buildbot installation and before you 450 restart the buildmaster to use the new version. 451 452 If you have modified the files in your working directory, this command 453 will leave them untouched, but will put the new recommended contents in a 454 .new file (for example, if index.html has been modified, this command 455 will create index.html.new). You can then look at the new version and 456 decide how to merge its contents into your modified file. 457 """+DB_HELP+""" 458 When upgrading from a pre-0.8.0 release (which did not use a database), 459 this command will create the given database and migrate data from the old 460 pickle files into it, then move the pickle files out of the way (e.g. to 461 changes.pck.old). To revert to an older release, rename the pickle files 462 back. When you are satisfied with the new version, you can delete the old 463 pickle files. 464 """

465

466 @in_reactor 467 @defer.deferredGenerator 468 -def upgradeMaster(config):

469 m = Maker(config) 470 471 if not config['quiet']: print "upgrading basedir" 472 basedir = os.path.expanduser(config['basedir']) 473 # TODO: check Makefile 474 # TODO: check TAC file 475 # check web files: index.html, default.css, robots.txt 476 m.upgrade_public_html({ 477 'bg_gradient.jpg' : util.sibpath(__file__, "../status/web/files/bg_gradient.jpg"), 478 'default.css' : util.sibpath(__file__, "../status/web/files/default.css"), 479 'robots.txt' : util.sibpath(__file__, "../status/web/files/robots.txt"), 480 'favicon.ico' : util.sibpath(__file__, "../status/web/files/favicon.ico"), 481 }) 482 m.populate_if_missing(os.path.join(basedir, "master.cfg.sample"), 483 util.sibpath(__file__, "sample.cfg"), 484 overwrite=True) 485 # if index.html exists, use it to override the root page tempalte 486 m.move_if_present(os.path.join(basedir, "public_html/index.html"), 487 os.path.join(basedir, "templates/root.html")) 488 489 if not config['quiet']: print "checking master.cfg" 490 wfd = defer.waitForDeferred( 491 m.check_master_cfg(expected_db_url=config['db'])) 492 yield wfd 493 rc = wfd.getResult() 494 495 if rc == 0: 496 from buildbot.db import connector 497 from buildbot.master import BuildMaster 498 499 if not config['quiet']: print "upgrading database" 500 db = connector.DBConnector(BuildMaster(config['basedir']), 501 config['db'], 502 basedir=config['basedir']) 503 504 wfd = defer.waitForDeferred( 505 db.model.upgrade()) 506 yield wfd 507 wfd.getResult() 508 509 if not config['quiet']: print "upgrade complete" 510 yield 0 511 else: 512 yield rc

513

514 515 -class MasterOptions(MakerBase):

516 optFlags = [ 517 ["force", "f", 518 "Re-use an existing directory (will not overwrite master.cfg file)"], 519 ["relocatable", "r", 520 "Create a relocatable buildbot.tac"], 521 ["no-logrotate", "n", 522 "Do not permit buildmaster rotate logs by itself"] 523 ] 524 optParameters = [ 525 ["config", "c", "master.cfg", "name of the buildmaster config file"], 526 ["log-size", "s", "10000000", 527 "size at which to rotate twisted log files"], 528 ["log-count", "l", "10", 529 "limit the number of kept old twisted log files"], 530 ["db", None, "sqlite:///state.sqlite", 531 "which DB to use for scheduler/status state. See below for syntax."], 532 ]

533 - def getSynopsis(self):

534 return "Usage: buildbot create-master [options] [<basedir>]"

535 536 longdesc = """ 537 This command creates a buildmaster working directory and buildbot.tac file. 538 The master will live in <dir> and create various files there. If 539 --relocatable is given, then the resulting buildbot.tac file will be 540 written such that its containing directory is assumed to be the basedir. 541 This is generally a good idea. 542 543 At runtime, the master will read a configuration file (named 544 'master.cfg' by default) in its basedir. This file should contain python 545 code which eventually defines a dictionary named 'BuildmasterConfig'. 546 The elements of this dictionary are used to configure the Buildmaster. 547 See doc/config.xhtml for details about what can be controlled through 548 this interface. 549 """ + DB_HELP + """ 550 The --db string is stored verbatim in the buildbot.tac file, and 551 evaluated as 'buildbot start' time to pass a DBConnector instance into 552 the newly-created BuildMaster object. 553 """ 554

555 - def postOptions(self):

556 MakerBase.postOptions(self) 557 if not re.match('^\d+$', self['log-size']): 558 raise usage.UsageError("log-size parameter needs to be an int") 559 if not re.match('^\d+$', self['log-count']) and \ 560 self['log-count'] != 'None': 561 raise usage.UsageError("log-count parameter needs to be an int "+ 562 " or None")

563 564 565 masterTACTemplate = [""" 566 import os 567 568 from twisted.application import service 569 from buildbot.master import BuildMaster 570 571 basedir = r'%(basedir)s' 572 rotateLength = %(log-size)s 573 maxRotatedFiles = %(log-count)s 574 575 # if this is a relocatable tac file, get the directory containing the TAC 576 if basedir == '.': 577 import os.path 578 basedir = os.path.abspath(os.path.dirname(__file__)) 579 580 # note: this line is matched against to check that this is a buildmaster 581 # directory; do not edit it. 582 application = service.Application('buildmaster') 583 """, 584 """ 585 try: 586 from twisted.python.logfile import LogFile 587 from twisted.python.log import ILogObserver, FileLogObserver 588 logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength, 589 maxRotatedFiles=maxRotatedFiles) 590 application.setComponent(ILogObserver, FileLogObserver(logfile).emit) 591 except ImportError: 592 # probably not yet twisted 8.2.0 and beyond, can't set log yet 593 pass 594 """, 595 """ 596 configfile = r'%(config)s' 597 598 m = BuildMaster(basedir, configfile) 599 m.setServiceParent(application) 600 m.log_rotation.rotateLength = rotateLength 601 m.log_rotation.maxRotatedFiles = maxRotatedFiles 602 603 """]

604 605 @in_reactor 606 -def createMaster(config):

607 m = Maker(config) 608 m.mkdir() 609 m.chdir() 610 if config['relocatable']: 611 config['basedir'] = '.' 612 if config['no-logrotate']: 613 masterTAC = "".join([masterTACTemplate[0]] + masterTACTemplate[2:]) 614 else: 615 masterTAC = "".join(masterTACTemplate) 616 contents = masterTAC % config 617 m.makeTAC(contents) 618 m.sampleconfig(util.sibpath(__file__, "sample.cfg")) 619 m.public_html({ 620 'bg_gradient.jpg' : util.sibpath(__file__, "../status/web/files/bg_gradient.jpg"), 621 'default.css' : util.sibpath(__file__, "../status/web/files/default.css"), 622 'robots.txt' : util.sibpath(__file__, "../status/web/files/robots.txt"), 623 'favicon.ico' : util.sibpath(__file__, "../status/web/files/favicon.ico"), 624 }) 625 m.makefile() 626 d = m.create_db() 627 628 def print_status(r): 629 if not m.quiet: 630 print "buildmaster configured in %s" % m.basedir

631 d.addCallback(print_status) 632 return d 633

634 -def stop(config, signame="TERM", wait=False):

635 import signal 636 basedir = config['basedir'] 637 quiet = config['quiet'] 638 639 if not isBuildmasterDir(config['basedir']): 640 print "not a buildmaster directory" 641 sys.exit(1) 642 643 os.chdir(basedir) 644 try: 645 f = open("twistd.pid", "rt") 646 except: 647 raise BuildbotNotRunningError 648 pid = int(f.read().strip()) 649 signum = getattr(signal, "SIG"+signame) 650 timer = 0 651 try: 652 os.kill(pid, signum) 653 except OSError, e: 654 if e.errno != 3: 655 raise 656 657 if not wait: 658 if not quiet: 659 print "sent SIG%s to process" % signame 660 return 661 time.sleep(0.1) 662 while timer < 10: 663 # poll once per second until twistd.pid goes away, up to 10 seconds 664 try: 665 os.kill(pid, 0) 666 except OSError: 667 if not quiet: 668 print "buildbot process %d is dead" % pid 669 return 670 timer += 1 671 time.sleep(1) 672 if not quiet: 673 print "never saw process go away"

674

675 -def restart(config):

676 basedir = config['basedir'] 677 quiet = config['quiet'] 678 679 if not isBuildmasterDir(basedir): 680 print "not a buildmaster directory" 681 sys.exit(1) 682 683 from buildbot.scripts.startup import start 684 try: 685 stop(config, wait=True) 686 except BuildbotNotRunningError: 687 pass 688 if not quiet: 689 print "now restarting buildbot process.." 690 start(config)

691

692 693 -class StartOptions(MakerBase):

694 optFlags = [ 695 ['quiet', 'q', "Don't display startup log messages"], 696 ]

697 - def getSynopsis(self):

698 return "Usage: buildbot start [<basedir>]"

699

700 -class StopOptions(MakerBase):

701 - def getSynopsis(self):

702 return "Usage: buildbot stop [<basedir>]"

703

704 -class ReconfigOptions(MakerBase):

705 optFlags = [ 706 ['quiet', 'q', "Don't display log messages about reconfiguration"], 707 ]

708 - def getSynopsis(self):

709 return "Usage: buildbot reconfig [<basedir>]"

710

711 712 713 -class RestartOptions(MakerBase):

714 optFlags = [ 715 ['quiet', 'q', "Don't display startup log messages"], 716 ]

717 - def getSynopsis(self):

718 return "Usage: buildbot restart [<basedir>]"

719

720 -class DebugClientOptions(OptionsWithOptionsFile):

721 optFlags = [ 722 ['help', 'h', "Display this message"], 723 ] 724 optParameters = [ 725 ["master", "m", None, 726 "Location of the buildmaster's slaveport (host:port)"], 727 ["passwd", "p", None, "Debug password to use"], 728 ] 729 buildbotOptions = [ 730 [ 'debugMaster', 'passwd' ], 731 [ 'master', 'master' ], 732 ]

733 - def getSynopsis(self):

734 return "Usage: buildbot debugclient [options]"

735

736 - def parseArgs(self, *args):

737 if len(args) > 0: 738 self['master'] = args[0] 739 if len(args) > 1: 740 self['passwd'] = args[1] 741 if len(args) > 2: 742 raise usage.UsageError("I wasn't expecting so many arguments")

743

744 -def debugclient(config):

745 from buildbot.clients import debug 746 747 master = config.get('master') 748 if master is None: 749 raise usage.UsageError("master must be specified: on the command " 750 "line or in ~/.buildbot/options") 751 752 passwd = config.get('passwd') 753 if passwd is None: 754 raise usage.UsageError("passwd must be specified: on the command " 755 "line or in ~/.buildbot/options") 756 757 d = debug.DebugWidget(master, passwd) 758 d.run()

759

760 -class StatusClientOptions(OptionsWithOptionsFile):

761 optFlags = [ 762 ['help', 'h', "Display this message"], 763 ] 764 optParameters = [ 765 ["master", "m", None, 766 "Location of the buildmaster's status port (host:port)"], 767 ["username", "u", "statusClient", "Username performing the trial build"], 768 ["passwd", None, "clientpw", "password for PB authentication"], 769 ] 770 buildbotOptions = [ 771 [ 'masterstatus', 'master' ], 772 ] 773

774 - def parseArgs(self, *args):

775 if len(args) > 0: 776 self['master'] = args[0] 777 if len(args) > 1: 778 raise usage.UsageError("I wasn't expecting so many arguments")

779

780 -class StatusLogOptions(StatusClientOptions):

781 - def getSynopsis(self):

782 return "Usage: buildbot statuslog [options]"

783

784 -class StatusGuiOptions(StatusClientOptions):

785 - def getSynopsis(self):

786 return "Usage: buildbot statusgui [options]"

787

788 -def statuslog(config):

789 from buildbot.clients import base 790 master = config.get('master') 791 if master is None: 792 raise usage.UsageError("master must be specified: on the command " 793 "line or in ~/.buildbot/options") 794 passwd = config.get('passwd') 795 username = config.get('username') 796 c = base.TextClient(master, username=username, passwd=passwd) 797 c.run()

798

799 -def statusgui(config):

800 from buildbot.clients import gtkPanes 801 master = config.get('master') 802 if master is None: 803 raise usage.UsageError("master must be specified: on the command " 804 "line or in ~/.buildbot/options") 805 passwd = config.get('passwd') 806 username = config.get('username') 807 c = gtkPanes.GtkClient(master, username=username, passwd=passwd) 808 c.run()

809

810 -class SendChangeOptions(OptionsWithOptionsFile):

811 - def __init__(self):

812 OptionsWithOptionsFile.__init__(self) 813 self['properties'] = {}

814 815 optParameters = [ 816 ("master", "m", None, 817 "Location of the buildmaster's PBListener (host:port)"), 818 # deprecated in 0.8.3; remove in 0.8.5 (bug #1711) 819 ("username", "u", None, "deprecated name for --who"), 820 ("auth", "a", None, "Authentication token - username:password, or prompt for password"), 821 ("who", "W", None, "Author of the commit"), 822 ("repository", "R", '', "Repository specifier"), 823 ("project", "P", '', "Project specifier"), 824 ("branch", "b", None, "Branch specifier"), 825 ("category", "C", None, "Category of repository"), 826 ("revision", "r", None, "Revision specifier"), 827 ("revision_file", None, None, "Filename containing revision spec"), 828 ("property", "p", None, 829 "A property for the change, in the format: name:value"), 830 ("comments", "c", None, "log message"), 831 ("logfile", "F", None, 832 "Read the log messages from this file (- for stdin)"), 833 ("when", "w", None, "timestamp to use as the change time"), 834 ("revlink", "l", '', "Revision link (revlink)"), 835 ("encoding", "e", 'utf8', 836 "Encoding of other parameters (default utf8)"), 837 ] 838 839 buildbotOptions = [ 840 [ 'master', 'master' ], 841 [ 'who', 'who' ], 842 # deprecated in 0.8.3; remove in 0.8.5 (bug #1711) 843 [ 'username', 'username' ], 844 [ 'branch', 'branch' ], 845 [ 'category', 'category' ], 846 ] 847

848 - def getSynopsis(self):

849 return "Usage: buildbot sendchange [options] filenames.."

850 - def parseArgs(self, *args):

851 self['files'] = args

852 - def opt_property(self, property):

853 name,value = property.split(':') 854 self['properties'][name] = value

855

856 857 -def sendchange(config, runReactor=False):

858 """Send a single change to the buildmaster's PBChangeSource. The 859 connection will be drpoped as soon as the Change has been sent.""" 860 from buildbot.clients import sendchange 861 862 encoding = config.get('encoding', 'utf8') 863 who = config.get('who') 864 if not who and config.get('username'): 865 print "NOTE: --username/-u is deprecated: use --who/-W'" 866 who = config.get('username') 867 auth = config.get('auth') 868 master = config.get('master') 869 branch = config.get('branch') 870 category = config.get('category') 871 revision = config.get('revision') 872 properties = config.get('properties', {}) 873 repository = config.get('repository', '') 874 project = config.get('project', '') 875 revlink = config.get('revlink', '') 876 if config.get('when'): 877 when = float(config.get('when')) 878 else: 879 when = None 880 if config.get("revision_file"): 881 revision = open(config["revision_file"],"r").read() 882 883 comments = config.get('comments') 884 if not comments and config.get('logfile'): 885 if config['logfile'] == "-": 886 f = sys.stdin 887 else: 888 f = open(config['logfile'], "rt") 889 comments = f.read() 890 if comments is None: 891 comments = "" 892 893 files = config.get('files', ()) 894 895 # fix up the auth with a password if none was given 896 if not auth: 897 auth = 'change:changepw' 898 if ':' not in auth: 899 import getpass 900 pw = getpass.getpass("Enter password for '%s': " % auth) 901 auth = "%s:%s" % (auth, pw) 902 auth = auth.split(':', 1) 903 904 assert who, "you must provide a committer (--who)" 905 assert master, "you must provide the master location" 906 907 s = sendchange.Sender(master, auth, encoding=encoding) 908 d = s.send(branch, revision, comments, files, who=who, category=category, when=when, 909 properties=properties, repository=repository, project=project, 910 revlink=revlink) 911 912 if runReactor: 913 from twisted.internet import reactor 914 status = [True] 915 def printSuccess(_): 916 print "change sent successfully"

917 def failed(f): 918 status[0] = False 919 print "change NOT sent - something went wrong: " + str(f) 920 d.addCallbacks(printSuccess, failed) 921 d.addBoth(lambda _ : reactor.stop()) 922 reactor.run() 923 return status[0] 924 return d 925

926 927 -class ForceOptions(OptionsWithOptionsFile):

928 optParameters = [ 929 ["builder", None, None, "which Builder to start"], 930 ["branch", None, None, "which branch to build"], 931 ["revision", None, None, "which revision to build"], 932 ["reason", None, None, "the reason for starting the build"], 933 ] 934

935 - def parseArgs(self, *args):

936 args = list(args) 937 if len(args) > 0: 938 if self['builder'] is not None: 939 raise usage.UsageError("--builder provided in two ways") 940 self['builder'] = args.pop(0) 941 if len(args) > 0: 942 if self['reason'] is not None: 943 raise usage.UsageError("--reason provided in two ways") 944 self['reason'] = " ".join(args)

945

946 947 -class TryOptions(OptionsWithOptionsFile):

948 optParameters = [ 949 ["connect", "c", None, 950 "How to reach the buildmaster, either 'ssh' or 'pb'"], 951 # for ssh, use --host, --username, and --jobdir 952 ["host", None, None, 953 "Hostname (used by ssh) for the buildmaster"], 954 ["jobdir", None, None, 955 "Directory (on the buildmaster host) where try jobs are deposited"], 956 ["username", "u", None, 957 "Username performing the try build"], 958 # for PB, use --master, --username, and --passwd 959 ["master", "m", None, 960 "Location of the buildmaster's PBListener (host:port)"], 961 ["passwd", None, None, 962 "Password for PB authentication"], 963 ["who", "w", None, 964 "Who is responsible for the try build"], 965 ["comment", "C", None, 966 "A comment which can be used in notifications for this build"], 967 968 ["diff", None, None, 969 "Filename of a patch to use instead of scanning a local tree. " 970 "Use '-' for stdin."], 971 ["patchlevel", "p", 0, 972 "Number of slashes to remove from patch pathnames, " 973 "like the -p option to 'patch'"], 974 975 ["baserev", None, None, 976 "Base revision to use instead of scanning a local tree."], 977 978 ["vc", None, None, 979 "The VC system in use, one of: bzr, cvs, darcs, git, hg, " 980 "mtn, p4, svn"], 981 ["branch", None, None, 982 "The branch in use, for VC systems that can't figure it out " 983 "themselves"], 984 ["repository", None, None, 985 "Repository to use, instead of path to working directory."], 986 987 ["builder", "b", None, 988 "Run the trial build on this Builder. Can be used multiple times."], 989 ["properties", None, None, 990 "A set of properties made available in the build environment, " 991 "format is --properties=prop1=value1,prop2=value2,.. " 992 "option can be specified multiple times."], 993 994 ["topfile", None, None, 995 "Name of a file at the top of the tree, used to find the top. " 996 "Only needed for SVN and CVS."], 997 ["topdir", None, None, 998 "Path to the top of the working copy. Only needed for SVN and CVS."], 999 ] 1000 1001 optFlags = [ 1002 ["wait", None, 1003 "wait until the builds have finished"], 1004 ["dryrun", 'n', 1005 "Gather info, but don't actually submit."], 1006 ["get-builder-names", None, 1007 "Get the names of available builders. Doesn't submit anything. " 1008 "Only supported for 'pb' connections."], 1009 ["quiet", "q", 1010 "Don't print status of current builds while waiting."], 1011 ] 1012 1013 # Mapping of .buildbot/options names to command-line options 1014 buildbotOptions = [ 1015 [ 'try_connect', 'connect' ], 1016 #[ 'try_builders', 'builders' ], <-- handled in postOptions 1017 [ 'try_vc', 'vc' ], 1018 [ 'try_branch', 'branch' ], 1019 [ 'try_repository', 'repository' ], 1020 [ 'try_topdir', 'topdir' ], 1021 [ 'try_topfile', 'topfile' ], 1022 [ 'try_host', 'host' ], 1023 [ 'try_username', 'username' ], 1024 [ 'try_jobdir', 'jobdir' ], 1025 [ 'try_passwd', 'passwd' ], 1026 [ 'try_master', 'master' ], 1027 [ 'try_who', 'who' ], 1028 [ 'try_comment', 'comment' ], 1029 #[ 'try_wait', 'wait' ], <-- handled in postOptions 1030 #[ 'try_quiet', 'quiet' ], <-- handled in postOptions 1031 1032 # Deprecated command mappings from the quirky old days: 1033 [ 'try_masterstatus', 'master' ], 1034 [ 'try_dir', 'jobdir' ], 1035 [ 'try_password', 'passwd' ], 1036 ] 1037

1038 - def __init__(self):

1039 OptionsWithOptionsFile.__init__(self) 1040 self['builders'] = [] 1041 self['properties'] = {}

1042

1043 - def opt_builder(self, option):

1044 self['builders'].append(option)

1045

1046 - def opt_properties(self, option):

1047 # We need to split the value of this option into a dictionary of properties 1048 propertylist = option.split(",") 1049 for i in range(0,len(propertylist)): 1050 splitproperty = propertylist[i].split("=", 1) 1051 self['properties'][splitproperty[0]] = splitproperty[1]

1052

1053 - def opt_patchlevel(self, option):

1054 self['patchlevel'] = int(option)

1055

1056 - def getSynopsis(self):

1057 return "Usage: buildbot try [options]"

1058

1059 - def postOptions(self):

1060 opts = loadOptionsFile() 1061 if not self['builders']: 1062 self['builders'] = opts.get('try_builders', []) 1063 if opts.get('try_wait', False): 1064 self['wait'] = True 1065 if opts.get('try_quiet', False): 1066 self['quiet'] = True 1067 # get the global 'masterstatus' option if it's set and no master 1068 # was specified otherwise 1069 if not self['master']: 1070 self['master'] = opts.get('masterstatus', None)

1071

1072 -def doTry(config):

1073 from buildbot.clients import tryclient 1074 t = tryclient.Try(config) 1075 t.run()

1076

1077 -class TryServerOptions(OptionsWithOptionsFile):

1078 optParameters = [ 1079 ["jobdir", None, None, "the jobdir (maildir) for submitting jobs"], 1080 ]

1081 - def getSynopsis(self):

1082 return "Usage: buildbot tryserver [options]"

1083

1084 1085 -def doTryServer(config):

1086 try: 1087 from hashlib import md5 1088 assert md5 1089 except ImportError: 1090 # For Python 2.4 compatibility 1091 import md5 1092 jobdir = os.path.expanduser(config["jobdir"]) 1093 job = sys.stdin.read() 1094 # now do a 'safecat'-style write to jobdir/tmp, then move atomically to 1095 # jobdir/new . Rather than come up with a unique name randomly, I'm just 1096 # going to MD5 the contents and prepend a timestamp. 1097 timestring = "%d" % time.time() 1098 try: 1099 m = md5() 1100 except TypeError: 1101 # For Python 2.4 compatibility 1102 m = md5.new() 1103 m.update(job) 1104 jobhash = m.hexdigest() 1105 fn = "%s-%s" % (timestring, jobhash) 1106 tmpfile = os.path.join(jobdir, "tmp", fn) 1107 newfile = os.path.join(jobdir, "new", fn) 1108 f = open(tmpfile, "w") 1109 f.write(job) 1110 f.close() 1111 os.rename(tmpfile, newfile)

1112

1113 1114 -class CheckConfigOptions(OptionsWithOptionsFile):

1115 optFlags = [ 1116 ['quiet', 'q', "Don't display error messages or tracebacks"], 1117 ] 1118

1119 - def getSynopsis(self):

1120 return "Usage: buildbot checkconfig [configFile]\n" + \ 1121 " If not specified, 'master.cfg' will be used as 'configFile'"

1122

1123 - def parseArgs(self, *args):

1124 if len(args) >= 1: 1125 self['configFile'] = args[0] 1126 else: 1127 self['configFile'] = 'master.cfg'

1128

1129 1130 @in_reactor 1131 -def doCheckConfig(config):

1132 from buildbot.scripts.checkconfig import ConfigLoader 1133 quiet = config.get('quiet') 1134 configFileName = config.get('configFile') 1135 1136 if os.path.isdir(configFileName): 1137 cl = ConfigLoader(basedir=configFileName) 1138 else: 1139 cl = ConfigLoader(configFileName=configFileName) 1140 1141 d = cl.load() 1142 1143 def cb(r): 1144 if not quiet: 1145 print "Config file is good!" 1146 return True

1147 def eb(f): 1148 if not quiet: 1149 f.printTraceback() 1150 return False 1151 d.addCallbacks(cb, eb) 1152 1153 return d 1154

1155 -class Options(usage.Options):

1156 synopsis = "Usage: buildbot <command> [command options]" 1157 1158 subCommands = [ 1159 # the following are all admin commands 1160 ['create-master', None, MasterOptions, 1161 "Create and populate a directory for a new buildmaster"], 1162 ['upgrade-master', None, UpgradeMasterOptions, 1163 "Upgrade an existing buildmaster directory for the current version"], 1164 ['start', None, StartOptions, "Start a buildmaster"], 1165 ['stop', None, StopOptions, "Stop a buildmaster"], 1166 ['restart', None, RestartOptions, 1167 "Restart a buildmaster"], 1168 1169 ['reconfig', None, ReconfigOptions, 1170 "SIGHUP a buildmaster to make it re-read the config file"], 1171 ['sighup', None, ReconfigOptions, 1172 "SIGHUP a buildmaster to make it re-read the config file"], 1173 1174 ['sendchange', None, SendChangeOptions, 1175 "Send a change to the buildmaster"], 1176 1177 ['debugclient', None, DebugClientOptions, 1178 "Launch a small debug panel GUI"], 1179 1180 ['statuslog', None, StatusLogOptions, 1181 "Emit current builder status to stdout"], 1182 ['statusgui', None, StatusGuiOptions, 1183 "Display a small window showing current builder status"], 1184 1185 #['force', None, ForceOptions, "Run a build"], 1186 ['try', None, TryOptions, "Run a build with your local changes"], 1187 1188 ['tryserver', None, TryServerOptions, 1189 "buildmaster-side 'try' support function, not for users"], 1190 1191 ['checkconfig', None, CheckConfigOptions, 1192 "test the validity of a master.cfg config file"], 1193 1194 # TODO: 'watch' 1195 ] 1196

1197 - def opt_version(self):

1198 import buildbot 1199 print "Buildbot version: %s" % buildbot.version 1200 usage.Options.opt_version(self)

1201

1202 - def opt_verbose(self):

1203 from twisted.python import log 1204 log.startLogging(sys.stderr)

1205

1206 - def postOptions(self):

1207 if not hasattr(self, 'subOptions'): 1208 raise usage.UsageError("must specify a command")

1209

1210 1211 -def run():

1212 config = Options() 1213 try: 1214 config.parseOptions() 1215 except usage.error, e: 1216 print "%s: %s" % (sys.argv[0], e) 1217 print 1218 c = getattr(config, 'subOptions', config) 1219 print str(c) 1220 sys.exit(1) 1221 1222 command = config.subCommand 1223 so = config.subOptions 1224 1225 if command == "create-master": 1226 createMaster(so) 1227 elif command == "upgrade-master": 1228 upgradeMaster(so) 1229 elif command == "start": 1230 from buildbot.scripts.startup import start 1231 1232 if not isBuildmasterDir(so['basedir']): 1233 print "not a buildmaster directory" 1234 sys.exit(1) 1235 1236 start(so) 1237 elif command == "stop": 1238 try: 1239 stop(so, wait=True) 1240 except BuildbotNotRunningError: 1241 if not so['quiet']: 1242 print "buildmaster not running" 1243 sys.exit(0) 1244 1245 elif command == "restart": 1246 restart(so) 1247 elif command == "reconfig" or command == "sighup": 1248 from buildbot.scripts.reconfig import Reconfigurator 1249 Reconfigurator().run(so) 1250 elif command == "sendchange": 1251 if not sendchange(so, True): 1252 sys.exit(1) 1253 elif command == "debugclient": 1254 debugclient(so) 1255 elif command == "statuslog": 1256 statuslog(so) 1257 elif command == "statusgui": 1258 statusgui(so) 1259 elif command == "try": 1260 doTry(so) 1261 elif command == "tryserver": 1262 doTryServer(so) 1263 elif command == "checkconfig": 1264 if not doCheckConfig(so): 1265 sys.exit(1) 1266 sys.exit(0)

1267

Source Code for Module buildbot.scripts.runner