buildbot.scripts.runner

1 # -*- test-case-name: buildbot.test.test_runner -*- 2 3 # N.B.: don't import anything that might pull in a reactor yet. Some of our 4 # subcommands want to load modules that need the gtk reactor. 5 # 6 # Also don't forget to mirror your changes on command-line options in manual 7 # pages and texinfo documentation. 8 9 import os, sys, stat, re, time 10 import traceback 11 from twisted.python import usage, util, runtime 12 13 from buildbot.interfaces import BuildbotNotRunningError 14

15 -def isBuildmasterDir(dir):

16 buildbot_tac = os.path.join(dir, "buildbot.tac") 17 if not os.path.isfile(buildbot_tac): 18 print "no buildbot.tac" 19 return False 20 21 contents = open(buildbot_tac, "r").read() 22 return "Application('buildmaster')" in contents

23 24 # the create/start/stop commands should all be run as the same user, 25 # preferably a separate 'buildbot' account. 26 27 # Note that the terms 'options' and 'config' are used intechangeably here - in 28 # fact, they are intercanged several times. Caveat legator. 29

30 -class OptionsWithOptionsFile(usage.Options):

31 # subclasses should set this to a list-of-lists in order to source the 32 # .buildbot/options file. 33 # buildbotOptions = [ [ 'optfile-name', 'option-name' ], .. ] 34 buildbotOptions = None 35

36 - def __init__(self, *args):

37 # for options in self.buildbotOptions, optParameters, and the options 38 # file, change the default in optParameters *before* calling through 39 # to the parent constructor 40 41 if self.buildbotOptions: 42 optfile = loadOptionsFile() 43 for optfile_name, option_name in self.buildbotOptions: 44 for i in range(len(self.optParameters)): 45 if self.optParameters[i][0] == option_name and optfile_name in optfile: 46 self.optParameters[i][2] = optfile[optfile_name] 47 usage.Options.__init__(self, *args)

48

49 -def loadOptionsFile(filename="options", here=None, home=None):

50 """Find the .buildbot/FILENAME file. Crawl from the current directory up 51 towards the root, and also look in ~/.buildbot . The first directory 52 that's owned by the user and has the file we're looking for wins. Windows 53 skips the owned-by-user test. 54 55 @rtype: dict 56 @return: a dictionary of names defined in the options file. If no options 57 file was found, return an empty dict. 58 """ 59 60 if here is None: 61 here = os.getcwd() 62 here = os.path.abspath(here) 63 64 if home is None: 65 if runtime.platformType == 'win32': 66 # never trust env-vars, use the proper API 67 from win32com.shell import shellcon, shell 68 appdata = shell.SHGetFolderPath(0, shellcon.CSIDL_APPDATA, 0, 0) 69 home = os.path.join(appdata, "buildbot") 70 else: 71 home = os.path.expanduser("~/.buildbot") 72 73 searchpath = [] 74 toomany = 20 75 while True: 76 searchpath.append(os.path.join(here, ".buildbot")) 77 next = os.path.dirname(here) 78 if next == here: 79 break # we've hit the root 80 here = next 81 toomany -= 1 # just in case 82 if toomany == 0: 83 raise ValueError("Hey, I seem to have wandered up into the " 84 "infinite glories of the heavens. Oops.") 85 searchpath.append(home) 86 87 localDict = {} 88 89 for d in searchpath: 90 if os.path.isdir(d): 91 if runtime.platformType != 'win32': 92 if os.stat(d)[stat.ST_UID] != os.getuid(): 93 print "skipping %s because you don't own it" % d 94 continue # security, skip other people's directories 95 optfile = os.path.join(d, filename) 96 if os.path.exists(optfile): 97 try: 98 f = open(optfile, "r") 99 options = f.read() 100 exec options in localDict 101 except: 102 print "error while reading %s" % optfile 103 raise 104 break 105 106 for k in localDict.keys(): 107 if k.startswith("__"): 108 del localDict[k] 109 return localDict

110

111 -class MakerBase(OptionsWithOptionsFile):

112 optFlags = [ 113 ['help', 'h', "Display this message"], 114 ["quiet", "q", "Do not emit the commands being run"], 115 ] 116 117 longdesc = """ 118 Operates upon the specified <basedir> (or the current directory, if not 119 specified). 120 """ 121 122 opt_h = usage.Options.opt_help 123

124 - def parseArgs(self, *args):

125 if len(args) > 0: 126 self['basedir'] = args[0] 127 else: 128 # Use the current directory if no basedir was specified. 129 self['basedir'] = os.getcwd() 130 if len(args) > 1: 131 raise usage.UsageError("I wasn't expecting so many arguments")

132

133 - def postOptions(self):

134 self['basedir'] = os.path.abspath(self['basedir'])

135 136 makefile_sample = """# -*- makefile -*- 137 138 # This is a simple makefile which lives in a buildmaster 139 # directory (next to the buildbot.tac file). It allows you to start/stop the 140 # master by doing 'make start' or 'make stop'. 141 142 # The 'reconfig' target will tell a buildmaster to reload its config file. 143 144 start: 145 twistd --no_save -y buildbot.tac 146 147 stop: 148 if [ -e twistd.pid ]; \\ 149 then kill `cat twistd.pid`; \\ 150 else echo "Nothing to stop."; \\ 151 fi 152 153 reconfig: 154 if [ -e twistd.pid ]; \\ 155 then kill -HUP `cat twistd.pid`; \\ 156 else echo "Nothing to reconfig."; \\ 157 fi 158 159 log: 160 if [ -e twistd.log ]; \\ 161 then tail -f twistd.log; \\ 162 else echo "Nothing to tail."; \\ 163 fi 164 """ 165

166 -class Maker:

167 - def __init__(self, config):

168 self.config = config 169 self.basedir = config['basedir'] 170 self.force = config.get('force', False) 171 self.quiet = config['quiet']

172

173 - def mkdir(self):

174 if os.path.exists(self.basedir): 175 if not self.quiet: 176 print "updating existing installation" 177 return 178 if not self.quiet: print "mkdir", self.basedir 179 os.mkdir(self.basedir)

180

181 - def mkinfo(self):

182 path = os.path.join(self.basedir, "info") 183 if not os.path.exists(path): 184 if not self.quiet: print "mkdir", path 185 os.mkdir(path) 186 created = False 187 admin = os.path.join(path, "admin") 188 if not os.path.exists(admin): 189 if not self.quiet: 190 print "Creating info/admin, you need to edit it appropriately" 191 f = open(admin, "wt") 192 f.write("Your Name Here <admin@youraddress.invalid>\n") 193 f.close() 194 created = True 195 host = os.path.join(path, "host") 196 if not os.path.exists(host): 197 if not self.quiet: 198 print "Creating info/host, you need to edit it appropriately" 199 f = open(host, "wt") 200 f.write("Please put a description of this build host here\n") 201 f.close() 202 created = True 203 access_uri = os.path.join(path, "access_uri") 204 if not os.path.exists(access_uri): 205 if not self.quiet: 206 print "Not creating info/access_uri - add it if you wish" 207 if created and not self.quiet: 208 print "Please edit the files in %s appropriately." % path

209

210 - def chdir(self):

211 if not self.quiet: print "chdir", self.basedir 212 os.chdir(self.basedir)

213

214 - def makeTAC(self, contents, secret=False):

215 tacfile = "buildbot.tac" 216 if os.path.exists(tacfile): 217 oldcontents = open(tacfile, "rt").read() 218 if oldcontents == contents: 219 if not self.quiet: 220 print "buildbot.tac already exists and is correct" 221 return 222 if not self.quiet: 223 print "not touching existing buildbot.tac" 224 print "creating buildbot.tac.new instead" 225 tacfile = "buildbot.tac.new" 226 f = open(tacfile, "wt") 227 f.write(contents) 228 f.close() 229 if secret: 230 os.chmod(tacfile, 0600)

231

232 - def makefile(self):

233 target = "Makefile.sample" 234 if os.path.exists(target): 235 oldcontents = open(target, "rt").read() 236 if oldcontents == makefile_sample: 237 if not self.quiet: 238 print "Makefile.sample already exists and is correct" 239 return 240 if not self.quiet: 241 print "replacing Makefile.sample" 242 else: 243 if not self.quiet: 244 print "creating Makefile.sample" 245 f = open(target, "wt") 246 f.write(makefile_sample) 247 f.close()

248

249 - def sampleconfig(self, source):

250 target = "master.cfg.sample" 251 config_sample = open(source, "rt").read() 252 if os.path.exists(target): 253 oldcontents = open(target, "rt").read() 254 if oldcontents == config_sample: 255 if not self.quiet: 256 print "master.cfg.sample already exists and is up-to-date" 257 return 258 if not self.quiet: 259 print "replacing master.cfg.sample" 260 else: 261 if not self.quiet: 262 print "creating master.cfg.sample" 263 f = open(target, "wt") 264 f.write(config_sample) 265 f.close() 266 os.chmod(target, 0600)

267

268 - def public_html(self, files):

269 webdir = os.path.join(self.basedir, "public_html") 270 if os.path.exists(webdir): 271 if not self.quiet: 272 print "public_html/ already exists: not replacing" 273 return 274 else: 275 os.mkdir(webdir) 276 if not self.quiet: 277 print "populating public_html/" 278 for target, source in files.iteritems(): 279 target = os.path.join(webdir, target) 280 f = open(target, "wt") 281 f.write(open(source, "rt").read()) 282 f.close()

283

284 - def create_db(self):

285 from buildbot.db import dbspec, exceptions 286 spec = dbspec.DBSpec.from_url(self.config["db"], self.basedir) 287 if not self.config['quiet']: print "creating database" 288 289 # upgrade from "nothing" 290 from buildbot.db.schema import manager 291 sm = manager.DBSchemaManager(spec, self.basedir) 292 if sm.get_db_version() != 0: 293 raise exceptions.DBAlreadyExistsError 294 sm.upgrade()

295

296 - def populate_if_missing(self, target, source, overwrite=False):

297 new_contents = open(source, "rt").read() 298 if os.path.exists(target): 299 old_contents = open(target, "rt").read() 300 if old_contents != new_contents: 301 if overwrite: 302 if not self.quiet: 303 print "%s has old/modified contents" % target 304 print " overwriting it with new contents" 305 open(target, "wt").write(new_contents) 306 else: 307 if not self.quiet: 308 print "%s has old/modified contents" % target 309 print " writing new contents to %s.new" % target 310 open(target + ".new", "wt").write(new_contents) 311 # otherwise, it's up to date 312 else: 313 if not self.quiet: 314 print "populating %s" % target 315 open(target, "wt").write(new_contents)

316

317 - def move_if_present(self, source, dest):

318 if os.path.exists(source): 319 if os.path.exists(dest): 320 print "Notice: %s now overrides %s" % (dest, source) 321 print " as the latter is not used by buildbot anymore." 322 print " Decide which one you want to keep." 323 else: 324 try: 325 print "Notice: Moving %s to %s." % (source, dest) 326 print " You can (and probably want to) remove it if you haven't modified this file." 327 os.renames(source, dest) 328 except Exception, e: 329 print "Error moving %s to %s: %s" % (source, dest, str(e))

330

331 - def upgrade_public_html(self, files):

332 webdir = os.path.join(self.basedir, "public_html") 333 if not os.path.exists(webdir): 334 if not self.quiet: 335 print "populating public_html/" 336 os.mkdir(webdir) 337 for target, source in files.iteritems(): 338 self.populate_if_missing(os.path.join(webdir, target), 339 source)

340

341 - def check_master_cfg(self):

342 from buildbot.master import BuildMaster 343 from twisted.python import log, failure 344 345 master_cfg = os.path.join(self.basedir, "master.cfg") 346 if not os.path.exists(master_cfg): 347 if not self.quiet: 348 print "No master.cfg found" 349 return 1 350 351 # side-effects of loading the config file: 352 353 # for each Builder defined in c['builders'], if the status directory 354 # didn't already exist, it will be created, and the 355 # $BUILDERNAME/builder pickle might be created (with a single 356 # "builder created" event). 357 358 # we put basedir in front of sys.path, because that's how the 359 # buildmaster itself will run, and it is quite common to have the 360 # buildmaster import helper classes from other .py files in its 361 # basedir. 362 363 if sys.path[0] != self.basedir: 364 sys.path.insert(0, self.basedir) 365 366 m = BuildMaster(self.basedir) 367 # we need to route log.msg to stdout, so any problems can be seen 368 # there. But if everything goes well, I'd rather not clutter stdout 369 # with log messages. So instead we add a logObserver which gathers 370 # messages and only displays them if something goes wrong. 371 messages = [] 372 log.addObserver(messages.append) 373 try: 374 # this will raise an exception if there's something wrong with 375 # the config file. Note that this BuildMaster instance is never 376 # started, so it won't actually do anything with the 377 # configuration. 378 m.loadConfig(open(master_cfg, "r"), check_synchronously_only=True) 379 except: 380 f = failure.Failure() 381 if not self.quiet: 382 print 383 for m in messages: 384 print "".join(m['message']) 385 print f 386 print 387 print "An error was detected in the master.cfg file." 388 print "Please correct the problem and run 'buildbot upgrade-master' again." 389 print 390 return 1 391 return 0

392 393 DB_HELP = """ 394 The --db string is evaluated to build the DB object, which specifies 395 which database the buildmaster should use to hold scheduler state and 396 status information. The default (which creates an SQLite database in 397 BASEDIR/state.sqlite) is equivalent to: 398 399 --db='DBSpec("sqlite3", basedir+"/state.sqlite"))' 400 --db='sqlite:///state.sqlite' 401 402 To use a remote MySQL database instead, use something like: 403 404 --db='mysql://bbuser:bbpasswd@dbhost/bbdb' 405 """ 406

407 -class UpgradeMasterOptions(MakerBase):

408 optFlags = [ 409 ["replace", "r", "Replace any modified files without confirmation."], 410 ] 411 optParameters = [ 412 ["db", None, "sqlite:///state.sqlite", 413 "which DB to use for scheduler/status state. See below for syntax."], 414 ] 415

416 - def getSynopsis(self):

417 return "Usage: buildbot upgrade-master [options] [<basedir>]"

418 419 longdesc = """ 420 This command takes an existing buildmaster working directory and 421 adds/modifies the files there to work with the current version of 422 buildbot. When this command is finished, the buildmaster directory should 423 look much like a brand-new one created by the 'create-master' command. 424 425 Use this after you've upgraded your buildbot installation and before you 426 restart the buildmaster to use the new version. 427 428 If you have modified the files in your working directory, this command 429 will leave them untouched, but will put the new recommended contents in a 430 .new file (for example, if index.html has been modified, this command 431 will create index.html.new). You can then look at the new version and 432 decide how to merge its contents into your modified file. 433 """+DB_HELP+""" 434 When upgrading from a pre-0.8.0 release (which did not use a database), 435 this command will create the given database and migrate data from the old 436 pickle files into it, then move the pickle files out of the way (e.g. to 437 changes.pck.old). To revert to an older release, rename the pickle files 438 back. When you are satisfied with the new version, you can delete the old 439 pickle files. 440 """

441

442 -def upgradeMaster(config):

443 basedir = os.path.expanduser(config['basedir']) 444 m = Maker(config) 445 # TODO: check Makefile 446 # TODO: check TAC file 447 # check web files: index.html, default.css, robots.txt 448 m.upgrade_public_html({ 449 'bg_gradient.jpg' : util.sibpath(__file__, "../status/web/files/bg_gradient.jpg"), 450 'default.css' : util.sibpath(__file__, "../status/web/files/default.css"), 451 'robots.txt' : util.sibpath(__file__, "../status/web/files/robots.txt"), 452 'favicon.ico' : util.sibpath(__file__, "../status/web/files/favicon.ico"), 453 }) 454 m.populate_if_missing(os.path.join(basedir, "master.cfg.sample"), 455 util.sibpath(__file__, "sample.cfg"), 456 overwrite=True) 457 # if index.html exists, use it to override the root page tempalte 458 m.move_if_present(os.path.join(basedir, "public_html/index.html"), 459 os.path.join(basedir, "templates/root.html")) 460 461 from buildbot.db import connector, dbspec 462 spec = dbspec.DBSpec.from_url(config["db"], basedir) 463 # TODO: check that TAC file specifies the right spec 464 465 # upgrade the db 466 from buildbot.db.schema import manager 467 sm = manager.DBSchemaManager(spec, basedir) 468 sm.upgrade() 469 470 # check the configuration 471 rc = m.check_master_cfg() 472 if rc: 473 return rc 474 if not config['quiet']: print "upgrade complete" 475 return 0

476 477

478 -class MasterOptions(MakerBase):

479 optFlags = [ 480 ["force", "f", 481 "Re-use an existing directory (will not overwrite master.cfg file)"], 482 ["relocatable", "r", 483 "Create a relocatable buildbot.tac"], 484 ["no-logrotate", "n", 485 "Do not permit buildmaster rotate logs by itself"] 486 ] 487 optParameters = [ 488 ["config", "c", "master.cfg", "name of the buildmaster config file"], 489 ["log-size", "s", "10000000", 490 "size at which to rotate twisted log files"], 491 ["log-count", "l", "10", 492 "limit the number of kept old twisted log files"], 493 ["db", None, "sqlite:///state.sqlite", 494 "which DB to use for scheduler/status state. See below for syntax."], 495 ]

496 - def getSynopsis(self):

497 return "Usage: buildbot create-master [options] [<basedir>]"

498 499 longdesc = """ 500 This command creates a buildmaster working directory and buildbot.tac file. 501 The master will live in <dir> and create various files there. If 502 --relocatable is given, then the resulting buildbot.tac file will be 503 written such that its containing directory is assumed to be the basedir. 504 This is generally a good idea. 505 506 At runtime, the master will read a configuration file (named 507 'master.cfg' by default) in its basedir. This file should contain python 508 code which eventually defines a dictionary named 'BuildmasterConfig'. 509 The elements of this dictionary are used to configure the Buildmaster. 510 See doc/config.xhtml for details about what can be controlled through 511 this interface. 512 """ + DB_HELP + """ 513 The --db string is stored verbatim in the buildbot.tac file, and 514 evaluated as 'buildbot start' time to pass a DBConnector instance into 515 the newly-created BuildMaster object. 516 """ 517

518 - def postOptions(self):

519 MakerBase.postOptions(self) 520 if not re.match('^\d+$', self['log-size']): 521 raise usage.UsageError("log-size parameter needs to be an int") 522 if not re.match('^\d+$', self['log-count']) and \ 523 self['log-count'] != 'None': 524 raise usage.UsageError("log-count parameter needs to be an int "+ 525 " or None")

526 527 528 masterTACTemplate = [""" 529 import os 530 531 from twisted.application import service 532 from buildbot.master import BuildMaster 533 534 basedir = r'%(basedir)s' 535 rotateLength = %(log-size)s 536 maxRotatedFiles = %(log-count)s 537 538 # if this is a relocatable tac file, get the directory containing the TAC 539 if basedir == '.': 540 import os.path 541 basedir = os.path.abspath(os.path.dirname(__file__)) 542 543 # note: this line is matched against to check that this is a buildmaster 544 # directory; do not edit it. 545 application = service.Application('buildmaster') 546 """, 547 """ 548 try: 549 from twisted.python.logfile import LogFile 550 from twisted.python.log import ILogObserver, FileLogObserver 551 logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength, 552 maxRotatedFiles=maxRotatedFiles) 553 application.setComponent(ILogObserver, FileLogObserver(logfile).emit) 554 except ImportError: 555 # probably not yet twisted 8.2.0 and beyond, can't set log yet 556 pass 557 """, 558 """ 559 configfile = r'%(config)s' 560 561 m = BuildMaster(basedir, configfile) 562 m.setServiceParent(application) 563 m.log_rotation.rotateLength = rotateLength 564 m.log_rotation.maxRotatedFiles = maxRotatedFiles 565 566 """] 567

568 -def createMaster(config):

569 m = Maker(config) 570 m.mkdir() 571 m.chdir() 572 if config['relocatable']: 573 config['basedir'] = '.' 574 if config['no-logrotate']: 575 masterTAC = "".join([masterTACTemplate[0]] + masterTACTemplate[2:]) 576 else: 577 masterTAC = "".join(masterTACTemplate) 578 contents = masterTAC % config 579 m.makeTAC(contents) 580 m.sampleconfig(util.sibpath(__file__, "sample.cfg")) 581 m.public_html({ 582 'bg_gradient.jpg' : util.sibpath(__file__, "../status/web/files/bg_gradient.jpg"), 583 'default.css' : util.sibpath(__file__, "../status/web/files/default.css"), 584 'robots.txt' : util.sibpath(__file__, "../status/web/files/robots.txt"), 585 'favicon.ico' : util.sibpath(__file__, "../status/web/files/favicon.ico"), 586 }) 587 m.makefile() 588 m.create_db() 589 590 if not m.quiet: print "buildmaster configured in %s" % m.basedir

591

592 -def stop(config, signame="TERM", wait=False):

593 import signal 594 basedir = config['basedir'] 595 quiet = config['quiet'] 596 597 if not isBuildmasterDir(config['basedir']): 598 print "not a buildmaster directory" 599 sys.exit(1) 600 601 os.chdir(basedir) 602 try: 603 f = open("twistd.pid", "rt") 604 except: 605 raise BuildbotNotRunningError 606 pid = int(f.read().strip()) 607 signum = getattr(signal, "SIG"+signame) 608 timer = 0 609 try: 610 os.kill(pid, signum) 611 except OSError, e: 612 if e.errno != 3: 613 raise 614 615 if not wait: 616 if not quiet: 617 print "sent SIG%s to process" % signame 618 return 619 time.sleep(0.1) 620 while timer < 10: 621 # poll once per second until twistd.pid goes away, up to 10 seconds 622 try: 623 os.kill(pid, 0) 624 except OSError: 625 if not quiet: 626 print "buildbot process %d is dead" % pid 627 return 628 timer += 1 629 time.sleep(1) 630 if not quiet: 631 print "never saw process go away"

632

633 -def restart(config):

634 basedir = config['basedir'] 635 quiet = config['quiet'] 636 637 if not isBuildmasterDir(config['basedir']): 638 print "not a buildmaster directory" 639 sys.exit(1) 640 641 from buildbot.scripts.startup import start 642 try: 643 stop(config, wait=True) 644 except BuildbotNotRunningError: 645 pass 646 if not quiet: 647 print "now restarting buildbot process.." 648 start(config)

649 650

651 -class StartOptions(MakerBase):

652 optFlags = [ 653 ['quiet', 'q', "Don't display startup log messages"], 654 ]

655 - def getSynopsis(self):

656 return "Usage: buildbot start [<basedir>]"

657

658 -class StopOptions(MakerBase):

659 - def getSynopsis(self):

660 return "Usage: buildbot stop [<basedir>]"

661

662 -class ReconfigOptions(MakerBase):

663 optFlags = [ 664 ['quiet', 'q', "Don't display log messages about reconfiguration"], 665 ]

666 - def getSynopsis(self):

667 return "Usage: buildbot reconfig [<basedir>]"

668 669 670

671 -class RestartOptions(MakerBase):

672 optFlags = [ 673 ['quiet', 'q', "Don't display startup log messages"], 674 ]

675 - def getSynopsis(self):

676 return "Usage: buildbot restart [<basedir>]"

677

678 -class DebugClientOptions(OptionsWithOptionsFile):

679 optFlags = [ 680 ['help', 'h', "Display this message"], 681 ] 682 optParameters = [ 683 ["master", "m", None, 684 "Location of the buildmaster's slaveport (host:port)"], 685 ["passwd", "p", None, "Debug password to use"], 686 ] 687 buildbotOptions = [ 688 [ 'debugMaster', 'passwd' ], 689 [ 'master', 'master' ], 690 ]

691 - def getSynopsis(self):

692 return "Usage: buildbot debugclient [options]"

693

694 - def parseArgs(self, *args):

695 if len(args) > 0: 696 self['master'] = args[0] 697 if len(args) > 1: 698 self['passwd'] = args[1] 699 if len(args) > 2: 700 raise usage.UsageError("I wasn't expecting so many arguments")

701

702 -def debugclient(config):

703 from buildbot.clients import debug 704 705 master = config.get('master') 706 if master is None: 707 raise usage.UsageError("master must be specified: on the command " 708 "line or in ~/.buildbot/options") 709 710 passwd = config.get('passwd') 711 if passwd is None: 712 raise usage.UsageError("passwd must be specified: on the command " 713 "line or in ~/.buildbot/options") 714 715 d = debug.DebugWidget(master, passwd) 716 d.run()

717

718 -class StatusClientOptions(OptionsWithOptionsFile):

719 optFlags = [ 720 ['help', 'h', "Display this message"], 721 ] 722 optParameters = [ 723 ["master", "m", None, 724 "Location of the buildmaster's status port (host:port)"], 725 ["username", "u", "statusClient", "Username performing the trial build"], 726 ["passwd", None, "clientpw", "password for PB authentication"], 727 ] 728 buildbotOptions = [ 729 [ 'masterstatus', 'master' ], 730 ] 731

732 - def parseArgs(self, *args):

733 if len(args) > 0: 734 self['master'] = args[0] 735 if len(args) > 1: 736 raise usage.UsageError("I wasn't expecting so many arguments")

737

738 -class StatusLogOptions(StatusClientOptions):

739 - def getSynopsis(self):

740 return "Usage: buildbot statuslog [options]"

741

742 -class StatusGuiOptions(StatusClientOptions):

743 - def getSynopsis(self):

744 return "Usage: buildbot statusgui [options]"

745

746 -def statuslog(config):

747 from buildbot.clients import base 748 master = config.get('master') 749 if master is None: 750 raise usage.UsageError("master must be specified: on the command " 751 "line or in ~/.buildbot/options") 752 passwd = config.get('passwd') 753 username = config.get('username') 754 c = base.TextClient(master, username=username, passwd=passwd) 755 c.run()

756

757 -def statusgui(config):

758 from buildbot.clients import gtkPanes 759 master = config.get('master') 760 if master is None: 761 raise usage.UsageError("master must be specified: on the command " 762 "line or in ~/.buildbot/options") 763 c = gtkPanes.GtkClient(master) 764 c.run()

765

766 -class SendChangeOptions(OptionsWithOptionsFile):

767 - def __init__(self):

768 OptionsWithOptionsFile.__init__(self) 769 self['properties'] = {}

770 771 optParameters = [ 772 ("master", "m", None, 773 "Location of the buildmaster's PBListener (host:port)"), 774 ("username", "u", None, "Username performing the commit"), 775 ("repository", "R", None, "Repository specifier"), 776 ("project", "P", None, "Project specifier"), 777 ("branch", "b", None, "Branch specifier"), 778 ("category", "C", None, "Category of repository"), 779 ("revision", "r", None, "Revision specifier"), 780 ("revision_file", None, None, "Filename containing revision spec"), 781 ("property", "p", None, 782 "A property for the change, in the format: name:value"), 783 ("comments", "c", None, "log message"), 784 ("logfile", "F", None, 785 "Read the log messages from this file (- for stdin)"), 786 ("when", "w", None, "timestamp to use as the change time"), 787 ("revlink", "l", None, "Revision link (revlink)"), 788 ] 789 790 buildbotOptions = [ 791 [ 'master', 'master' ], 792 [ 'username', 'username' ], 793 [ 'branch', 'branch' ], 794 [ 'category', 'category' ], 795 ] 796

797 - def getSynopsis(self):

798 return "Usage: buildbot sendchange [options] filenames.."

799 - def parseArgs(self, *args):

800 self['files'] = args

801 - def opt_property(self, property):

802 name,value = property.split(':') 803 self['properties'][name] = value

804 805

806 -def sendchange(config, runReactor=False):

807 """Send a single change to the buildmaster's PBChangeSource. The 808 connection will be drpoped as soon as the Change has been sent.""" 809 from buildbot.clients.sendchange import Sender 810 811 user = config.get('username') 812 master = config.get('master') 813 branch = config.get('branch') 814 category = config.get('category') 815 revision = config.get('revision') 816 properties = config.get('properties', {}) 817 repository = config.get('repository', '') 818 project = config.get('project', '') 819 revlink = config.get('revlink', '') 820 if config.get('when'): 821 when = float(config.get('when')) 822 else: 823 when = None 824 if config.get("revision_file"): 825 revision = open(config["revision_file"],"r").read() 826 827 comments = config.get('comments') 828 if not comments and config.get('logfile'): 829 if config['logfile'] == "-": 830 f = sys.stdin 831 else: 832 f = open(config['logfile'], "rt") 833 comments = f.read() 834 if comments is None: 835 comments = "" 836 837 files = config.get('files', []) 838 839 assert user, "you must provide a username" 840 assert master, "you must provide the master location" 841 842 s = Sender(master, user) 843 d = s.send(branch, revision, comments, files, category=category, when=when, 844 properties=properties, repository=repository, project=project, 845 revlink=revlink) 846 if runReactor: 847 status = [True] 848 def failed(res): 849 status[0] = False 850 s.printFailure(res)

851 d.addCallbacks(s.printSuccess, failed) 852 d.addBoth(s.stop) 853 s.run() 854 return status[0] 855 return d 856 857

858 -class ForceOptions(OptionsWithOptionsFile):

859 optParameters = [ 860 ["builder", None, None, "which Builder to start"], 861 ["branch", None, None, "which branch to build"], 862 ["revision", None, None, "which revision to build"], 863 ["reason", None, None, "the reason for starting the build"], 864 ] 865

866 - def parseArgs(self, *args):

867 args = list(args) 868 if len(args) > 0: 869 if self['builder'] is not None: 870 raise usage.UsageError("--builder provided in two ways") 871 self['builder'] = args.pop(0) 872 if len(args) > 0: 873 if self['reason'] is not None: 874 raise usage.UsageError("--reason provided in two ways") 875 self['reason'] = " ".join(args)

876 877

878 -class TryOptions(OptionsWithOptionsFile):

879 optParameters = [ 880 ["connect", "c", None, 881 "how to reach the buildmaster, either 'ssh' or 'pb'"], 882 # for ssh, use --tryhost, --username, and --trydir 883 ["tryhost", None, None, 884 "the hostname (used by ssh) for the buildmaster"], 885 ["trydir", None, None, 886 "the directory (on the tryhost) where tryjobs are deposited"], 887 ["username", "u", None, "Username performing the trial build"], 888 # for PB, use --master, --username, and --passwd 889 ["master", "m", None, 890 "Location of the buildmaster's PBListener (host:port)"], 891 ["passwd", None, None, "password for PB authentication"], 892 893 ["diff", None, None, 894 "Filename of a patch to use instead of scanning a local tree. Use '-' for stdin."], 895 ["patchlevel", "p", 0, 896 "Number of slashes to remove from patch pathnames, like the -p option to 'patch'"], 897 898 ["baserev", None, None, 899 "Base revision to use instead of scanning a local tree."], 900 901 ["vc", None, None, 902 "The VC system in use, one of: cvs,svn,bzr,darcs,p4"], 903 ["branch", None, None, 904 "The branch in use, for VC systems that can't figure it out" 905 " themselves"], 906 907 ["builder", "b", None, 908 "Run the trial build on this Builder. Can be used multiple times."], 909 ["properties", None, None, 910 "A set of properties made available in the build environment, format:prop1=value1,prop2=value2..."], 911 912 ["try-topfile", None, None, 913 "Name of a file at the top of the tree, used to find the top. Only needed for SVN and CVS."], 914 ["try-topdir", None, None, 915 "Path to the top of the working copy. Only needed for SVN and CVS."], 916 917 ] 918 919 optFlags = [ 920 ["wait", None, "wait until the builds have finished"], 921 ["dryrun", 'n', "Gather info, but don't actually submit."], 922 ["get-builder-names", None, "Get the names of available builders. Doesn't submit anything. Only supported for 'pb' connections."], 923 ] 924 925 # here it is, the definitive, quirky mapping of .buildbot/options names to 926 # command-line options. Design by committee, anyone? 927 buildbotOptions = [ 928 [ 'try_connect', 'connect' ], 929 #[ 'try_builders', 'builders' ], <-- handled in postOptions 930 [ 'try_vc', 'vc' ], 931 [ 'try_branch', 'branch' ], 932 [ 'try_topdir', 'try-topdir' ], 933 [ 'try_topfile', 'try-topfile' ], 934 [ 'try_host', 'tryhost' ], 935 [ 'try_username', 'username' ], 936 [ 'try_dir', 'trydir' ], 937 [ 'try_password', 'passwd' ], 938 [ 'try_master', 'master' ], 939 #[ 'try_wait', 'wait' ], <-- handled in postOptions 940 [ 'masterstatus', 'master' ], 941 ] 942

943 - def __init__(self):

944 OptionsWithOptionsFile.__init__(self) 945 self['builders'] = [] 946 self['properties'] = {}

947

948 - def opt_builder(self, option):

949 self['builders'].append(option)

950

951 - def opt_properties(self, option):

952 # We need to split the value of this option into a dictionary of properties 953 properties = {} 954 propertylist = option.split(",") 955 for i in range(0,len(propertylist)): 956 print propertylist[i] 957 splitproperty = propertylist[i].split("=") 958 properties[splitproperty[0]] = splitproperty[1] 959 self['properties'] = properties

960

961 - def opt_patchlevel(self, option):

962 self['patchlevel'] = int(option)

963

964 - def getSynopsis(self):

965 return "Usage: buildbot try [options]"

966

967 - def postOptions(self):

968 opts = loadOptionsFile() 969 if not self['builders']: 970 self['builders'] = opts.get('try_builders', []) 971 if opts.get('try_wait', False): 972 self['wait'] = True

973

974 -def doTry(config):

975 from buildbot.clients import tryclient 976 t = tryclient.Try(config) 977 t.run()

978

979 -class TryServerOptions(OptionsWithOptionsFile):

980 optParameters = [ 981 ["jobdir", None, None, "the jobdir (maildir) for submitting jobs"], 982 ]

983 - def getSynopsis(self):

984 return "Usage: buildbot tryserver [options]"

985 986

987 -def doTryServer(config):

988 try: 989 from hashlib import md5 990 except ImportError: 991 # For Python 2.4 compatibility 992 import md5 993 jobdir = os.path.expanduser(config["jobdir"]) 994 job = sys.stdin.read() 995 # now do a 'safecat'-style write to jobdir/tmp, then move atomically to 996 # jobdir/new . Rather than come up with a unique name randomly, I'm just 997 # going to MD5 the contents and prepend a timestamp. 998 timestring = "%d" % time.time() 999 try: 1000 m = md5() 1001 except TypeError: 1002 # For Python 2.4 compatibility 1003 m = md5.new() 1004 m.update(job) 1005 jobhash = m.hexdigest() 1006 fn = "%s-%s" % (timestring, jobhash) 1007 tmpfile = os.path.join(jobdir, "tmp", fn) 1008 newfile = os.path.join(jobdir, "new", fn) 1009 f = open(tmpfile, "w") 1010 f.write(job) 1011 f.close() 1012 os.rename(tmpfile, newfile)

1013 1014

1015 -class CheckConfigOptions(OptionsWithOptionsFile):

1016 optFlags = [ 1017 ['quiet', 'q', "Don't display error messages or tracebacks"], 1018 ] 1019

1020 - def getSynopsis(self):

1021 return "Usage: buildbot checkconfig [configFile]\n" + \ 1022 " If not specified, 'master.cfg' will be used as 'configFile'"

1023

1024 - def parseArgs(self, *args):

1025 if len(args) >= 1: 1026 self['configFile'] = args[0] 1027 else: 1028 self['configFile'] = 'master.cfg'

1029 1030

1031 -def doCheckConfig(config):

1032 quiet = config.get('quiet') 1033 configFileName = config.get('configFile') 1034 try: 1035 from buildbot.scripts.checkconfig import ConfigLoader 1036 if os.path.isdir(configFileName): 1037 ConfigLoader(basedir=configFileName) 1038 else: 1039 ConfigLoader(configFileName=configFileName) 1040 except: 1041 if not quiet: 1042 # Print out the traceback in a nice format 1043 t, v, tb = sys.exc_info() 1044 traceback.print_exception(t, v, tb) 1045 sys.exit(1) 1046 1047 if not quiet: 1048 print "Config file is good!"

1049 1050

1051 -class Options(usage.Options):

1052 synopsis = "Usage: buildbot <command> [command options]" 1053 1054 subCommands = [ 1055 # the following are all admin commands 1056 ['create-master', None, MasterOptions, 1057 "Create and populate a directory for a new buildmaster"], 1058 ['upgrade-master', None, UpgradeMasterOptions, 1059 "Upgrade an existing buildmaster directory for the current version"], 1060 ['start', None, StartOptions, "Start a buildmaster"], 1061 ['stop', None, StopOptions, "Stop a buildmaster"], 1062 ['restart', None, RestartOptions, 1063 "Restart a buildmaster"], 1064 1065 ['reconfig', None, ReconfigOptions, 1066 "SIGHUP a buildmaster to make it re-read the config file"], 1067 ['sighup', None, ReconfigOptions, 1068 "SIGHUP a buildmaster to make it re-read the config file"], 1069 1070 ['sendchange', None, SendChangeOptions, 1071 "Send a change to the buildmaster"], 1072 1073 ['debugclient', None, DebugClientOptions, 1074 "Launch a small debug panel GUI"], 1075 1076 ['statuslog', None, StatusLogOptions, 1077 "Emit current builder status to stdout"], 1078 ['statusgui', None, StatusGuiOptions, 1079 "Display a small window showing current builder status"], 1080 1081 #['force', None, ForceOptions, "Run a build"], 1082 ['try', None, TryOptions, "Run a build with your local changes"], 1083 1084 ['tryserver', None, TryServerOptions, 1085 "buildmaster-side 'try' support function, not for users"], 1086 1087 ['checkconfig', None, CheckConfigOptions, 1088 "test the validity of a master.cfg config file"], 1089 1090 # TODO: 'watch' 1091 ] 1092

1093 - def opt_version(self):

1094 import buildbot 1095 print "Buildbot version: %s" % buildbot.version 1096 usage.Options.opt_version(self)

1097

1098 - def opt_verbose(self):

1099 from twisted.python import log 1100 log.startLogging(sys.stderr)

1101

1102 - def postOptions(self):

1103 if not hasattr(self, 'subOptions'): 1104 raise usage.UsageError("must specify a command")

1105 1106

1107 -def run():

1108 config = Options() 1109 try: 1110 config.parseOptions() 1111 except usage.error, e: 1112 print "%s: %s" % (sys.argv[0], e) 1113 print 1114 c = getattr(config, 'subOptions', config) 1115 print str(c) 1116 sys.exit(1) 1117 1118 command = config.subCommand 1119 so = config.subOptions 1120 1121 if command == "create-master": 1122 createMaster(so) 1123 elif command == "upgrade-master": 1124 upgradeMaster(so) 1125 elif command == "start": 1126 from buildbot.scripts.startup import start 1127 1128 if not isBuildmasterDir(so['basedir']): 1129 print "not a buildmaster directory" 1130 sys.exit(1) 1131 1132 start(so) 1133 elif command == "stop": 1134 try: 1135 stop(so, wait=True) 1136 except BuildbotNotRunningError: 1137 if not so['quiet']: 1138 print "buildmaster not running" 1139 sys.exit(0) 1140 1141 elif command == "restart": 1142 restart(so) 1143 elif command == "reconfig" or command == "sighup": 1144 from buildbot.scripts.reconfig import Reconfigurator 1145 Reconfigurator().run(so) 1146 elif command == "sendchange": 1147 if not sendchange(so, True): 1148 sys.exit(1) 1149 elif command == "debugclient": 1150 debugclient(so) 1151 elif command == "statuslog": 1152 statuslog(so) 1153 elif command == "statusgui": 1154 statusgui(so) 1155 elif command == "try": 1156 doTry(so) 1157 elif command == "tryserver": 1158 doTryServer(so) 1159 elif command == "checkconfig": 1160 doCheckConfig(so) 1161 sys.exit(0)

1162

Source Code for Module buildbot.scripts.runner