buildbot.scripts.runner

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 17 # N.B.: don't import anything that might pull in a reactor yet. Some of our 18 # subcommands want to load modules that need the gtk reactor. 19 # 20 # Also don't forget to mirror your changes on command-line options in manual 21 # pages and texinfo documentation. 22 23 import os, sys, stat, re, time 24 import traceback 25 from twisted.python import usage, util, runtime 26 27 from buildbot.interfaces import BuildbotNotRunningError 28

29 -def isBuildmasterDir(dir):

30 buildbot_tac = os.path.join(dir, "buildbot.tac") 31 if not os.path.isfile(buildbot_tac): 32 print "no buildbot.tac" 33 return False 34 35 contents = open(buildbot_tac, "r").read() 36 return "Application('buildmaster')" in contents

37 38 # the create/start/stop commands should all be run as the same user, 39 # preferably a separate 'buildbot' account. 40 41 # Note that the terms 'options' and 'config' are used intechangeably here - in 42 # fact, they are intercanged several times. Caveat legator. 43

44 -class OptionsWithOptionsFile(usage.Options):

45 # subclasses should set this to a list-of-lists in order to source the 46 # .buildbot/options file. 47 # buildbotOptions = [ [ 'optfile-name', 'option-name' ], .. ] 48 buildbotOptions = None 49

50 - def __init__(self, *args):

51 # for options in self.buildbotOptions, optParameters, and the options 52 # file, change the default in optParameters *before* calling through 53 # to the parent constructor 54 55 if self.buildbotOptions: 56 optfile = loadOptionsFile() 57 for optfile_name, option_name in self.buildbotOptions: 58 for i in range(len(self.optParameters)): 59 if self.optParameters[i][0] == option_name and optfile_name in optfile: 60 self.optParameters[i][2] = optfile[optfile_name] 61 usage.Options.__init__(self, *args)

62

63 -def loadOptionsFile(filename="options", here=None, home=None):

64 """Find the .buildbot/FILENAME file. Crawl from the current directory up 65 towards the root, and also look in ~/.buildbot . The first directory 66 that's owned by the user and has the file we're looking for wins. Windows 67 skips the owned-by-user test. 68 69 @rtype: dict 70 @return: a dictionary of names defined in the options file. If no options 71 file was found, return an empty dict. 72 """ 73 74 if here is None: 75 here = os.getcwd() 76 here = os.path.abspath(here) 77 78 if home is None: 79 if runtime.platformType == 'win32': 80 # never trust env-vars, use the proper API 81 from win32com.shell import shellcon, shell 82 appdata = shell.SHGetFolderPath(0, shellcon.CSIDL_APPDATA, 0, 0) 83 home = os.path.join(appdata, "buildbot") 84 else: 85 home = os.path.expanduser("~/.buildbot") 86 87 searchpath = [] 88 toomany = 20 89 while True: 90 searchpath.append(os.path.join(here, ".buildbot")) 91 next = os.path.dirname(here) 92 if next == here: 93 break # we've hit the root 94 here = next 95 toomany -= 1 # just in case 96 if toomany == 0: 97 raise ValueError("Hey, I seem to have wandered up into the " 98 "infinite glories of the heavens. Oops.") 99 searchpath.append(home) 100 101 localDict = {} 102 103 for d in searchpath: 104 if os.path.isdir(d): 105 if runtime.platformType != 'win32': 106 if os.stat(d)[stat.ST_UID] != os.getuid(): 107 print "skipping %s because you don't own it" % d 108 continue # security, skip other people's directories 109 optfile = os.path.join(d, filename) 110 if os.path.exists(optfile): 111 try: 112 f = open(optfile, "r") 113 options = f.read() 114 exec options in localDict 115 except: 116 print "error while reading %s" % optfile 117 raise 118 break 119 120 for k in localDict.keys(): 121 if k.startswith("__"): 122 del localDict[k] 123 return localDict

124

125 -class MakerBase(OptionsWithOptionsFile):

126 optFlags = [ 127 ['help', 'h', "Display this message"], 128 ["quiet", "q", "Do not emit the commands being run"], 129 ] 130 131 longdesc = """ 132 Operates upon the specified <basedir> (or the current directory, if not 133 specified). 134 """ 135 136 opt_h = usage.Options.opt_help 137

138 - def parseArgs(self, *args):

139 if len(args) > 0: 140 self['basedir'] = args[0] 141 else: 142 # Use the current directory if no basedir was specified. 143 self['basedir'] = os.getcwd() 144 if len(args) > 1: 145 raise usage.UsageError("I wasn't expecting so many arguments")

146

147 - def postOptions(self):

148 self['basedir'] = os.path.abspath(self['basedir'])

149 150 makefile_sample = """# -*- makefile -*- 151 152 # This is a simple makefile which lives in a buildmaster 153 # directory (next to the buildbot.tac file). It allows you to start/stop the 154 # master by doing 'make start' or 'make stop'. 155 156 # The 'reconfig' target will tell a buildmaster to reload its config file. 157 158 start: 159 twistd --no_save -y buildbot.tac 160 161 stop: 162 if [ -e twistd.pid ]; \\ 163 then kill `cat twistd.pid`; \\ 164 else echo "Nothing to stop."; \\ 165 fi 166 167 reconfig: 168 if [ -e twistd.pid ]; \\ 169 then kill -HUP `cat twistd.pid`; \\ 170 else echo "Nothing to reconfig."; \\ 171 fi 172 173 log: 174 if [ -e twistd.log ]; \\ 175 then tail -f twistd.log; \\ 176 else echo "Nothing to tail."; \\ 177 fi 178 """ 179

180 -class Maker:

181 - def __init__(self, config):

182 self.config = config 183 self.basedir = config['basedir'] 184 self.force = config.get('force', False) 185 self.quiet = config['quiet']

186

187 - def mkdir(self):

188 if os.path.exists(self.basedir): 189 if not self.quiet: 190 print "updating existing installation" 191 return 192 if not self.quiet: print "mkdir", self.basedir 193 os.mkdir(self.basedir)

194

195 - def mkinfo(self):

196 path = os.path.join(self.basedir, "info") 197 if not os.path.exists(path): 198 if not self.quiet: print "mkdir", path 199 os.mkdir(path) 200 created = False 201 admin = os.path.join(path, "admin") 202 if not os.path.exists(admin): 203 if not self.quiet: 204 print "Creating info/admin, you need to edit it appropriately" 205 f = open(admin, "wt") 206 f.write("Your Name Here <admin@youraddress.invalid>\n") 207 f.close() 208 created = True 209 host = os.path.join(path, "host") 210 if not os.path.exists(host): 211 if not self.quiet: 212 print "Creating info/host, you need to edit it appropriately" 213 f = open(host, "wt") 214 f.write("Please put a description of this build host here\n") 215 f.close() 216 created = True 217 access_uri = os.path.join(path, "access_uri") 218 if not os.path.exists(access_uri): 219 if not self.quiet: 220 print "Not creating info/access_uri - add it if you wish" 221 if created and not self.quiet: 222 print "Please edit the files in %s appropriately." % path

223

224 - def chdir(self):

225 if not self.quiet: print "chdir", self.basedir 226 os.chdir(self.basedir)

227

228 - def makeTAC(self, contents, secret=False):

229 tacfile = "buildbot.tac" 230 if os.path.exists(tacfile): 231 oldcontents = open(tacfile, "rt").read() 232 if oldcontents == contents: 233 if not self.quiet: 234 print "buildbot.tac already exists and is correct" 235 return 236 if not self.quiet: 237 print "not touching existing buildbot.tac" 238 print "creating buildbot.tac.new instead" 239 tacfile = "buildbot.tac.new" 240 f = open(tacfile, "wt") 241 f.write(contents) 242 f.close() 243 if secret: 244 os.chmod(tacfile, 0600)

245

246 - def makefile(self):

247 target = "Makefile.sample" 248 if os.path.exists(target): 249 oldcontents = open(target, "rt").read() 250 if oldcontents == makefile_sample: 251 if not self.quiet: 252 print "Makefile.sample already exists and is correct" 253 return 254 if not self.quiet: 255 print "replacing Makefile.sample" 256 else: 257 if not self.quiet: 258 print "creating Makefile.sample" 259 f = open(target, "wt") 260 f.write(makefile_sample) 261 f.close()

262

263 - def sampleconfig(self, source):

264 target = "master.cfg.sample" 265 config_sample = open(source, "rt").read() 266 if os.path.exists(target): 267 oldcontents = open(target, "rt").read() 268 if oldcontents == config_sample: 269 if not self.quiet: 270 print "master.cfg.sample already exists and is up-to-date" 271 return 272 if not self.quiet: 273 print "replacing master.cfg.sample" 274 else: 275 if not self.quiet: 276 print "creating master.cfg.sample" 277 f = open(target, "wt") 278 f.write(config_sample) 279 f.close() 280 os.chmod(target, 0600)

281

282 - def public_html(self, files):

283 webdir = os.path.join(self.basedir, "public_html") 284 if os.path.exists(webdir): 285 if not self.quiet: 286 print "public_html/ already exists: not replacing" 287 return 288 else: 289 os.mkdir(webdir) 290 if not self.quiet: 291 print "populating public_html/" 292 for target, source in files.iteritems(): 293 target = os.path.join(webdir, target) 294 f = open(target, "wt") 295 f.write(open(source, "rt").read()) 296 f.close()

297

298 - def create_db(self):

299 from buildbot.db import dbspec, exceptions 300 spec = dbspec.DBSpec.from_url(self.config["db"], self.basedir) 301 if not self.config['quiet']: print "creating database" 302 303 # upgrade from "nothing" 304 from buildbot.db.schema import manager 305 sm = manager.DBSchemaManager(spec, self.basedir) 306 if sm.get_db_version() != 0: 307 raise exceptions.DBAlreadyExistsError 308 sm.upgrade()

309

310 - def populate_if_missing(self, target, source, overwrite=False):

311 new_contents = open(source, "rt").read() 312 if os.path.exists(target): 313 old_contents = open(target, "rt").read() 314 if old_contents != new_contents: 315 if overwrite: 316 if not self.quiet: 317 print "%s has old/modified contents" % target 318 print " overwriting it with new contents" 319 open(target, "wt").write(new_contents) 320 else: 321 if not self.quiet: 322 print "%s has old/modified contents" % target 323 print " writing new contents to %s.new" % target 324 open(target + ".new", "wt").write(new_contents) 325 # otherwise, it's up to date 326 else: 327 if not self.quiet: 328 print "populating %s" % target 329 open(target, "wt").write(new_contents)

330

331 - def move_if_present(self, source, dest):

332 if os.path.exists(source): 333 if os.path.exists(dest): 334 print "Notice: %s now overrides %s" % (dest, source) 335 print " as the latter is not used by buildbot anymore." 336 print " Decide which one you want to keep." 337 else: 338 try: 339 print "Notice: Moving %s to %s." % (source, dest) 340 print " You can (and probably want to) remove it if you haven't modified this file." 341 os.renames(source, dest) 342 except Exception, e: 343 print "Error moving %s to %s: %s" % (source, dest, str(e))

344

345 - def upgrade_public_html(self, files):

346 webdir = os.path.join(self.basedir, "public_html") 347 if not os.path.exists(webdir): 348 if not self.quiet: 349 print "populating public_html/" 350 os.mkdir(webdir) 351 for target, source in files.iteritems(): 352 self.populate_if_missing(os.path.join(webdir, target), 353 source)

354

355 - def check_master_cfg(self):

356 from buildbot.master import BuildMaster 357 from twisted.python import log, failure 358 359 master_cfg = os.path.join(self.basedir, "master.cfg") 360 if not os.path.exists(master_cfg): 361 if not self.quiet: 362 print "No master.cfg found" 363 return 1 364 365 # side-effects of loading the config file: 366 367 # for each Builder defined in c['builders'], if the status directory 368 # didn't already exist, it will be created, and the 369 # $BUILDERNAME/builder pickle might be created (with a single 370 # "builder created" event). 371 372 # we put basedir in front of sys.path, because that's how the 373 # buildmaster itself will run, and it is quite common to have the 374 # buildmaster import helper classes from other .py files in its 375 # basedir. 376 377 if sys.path[0] != self.basedir: 378 sys.path.insert(0, self.basedir) 379 380 m = BuildMaster(self.basedir) 381 # we need to route log.msg to stdout, so any problems can be seen 382 # there. But if everything goes well, I'd rather not clutter stdout 383 # with log messages. So instead we add a logObserver which gathers 384 # messages and only displays them if something goes wrong. 385 messages = [] 386 log.addObserver(messages.append) 387 try: 388 # this will raise an exception if there's something wrong with 389 # the config file. Note that this BuildMaster instance is never 390 # started, so it won't actually do anything with the 391 # configuration. 392 m.loadConfig(open(master_cfg, "r"), check_synchronously_only=True) 393 except: 394 f = failure.Failure() 395 if not self.quiet: 396 print 397 for m in messages: 398 print "".join(m['message']) 399 print f 400 print 401 print "An error was detected in the master.cfg file." 402 print "Please correct the problem and run 'buildbot upgrade-master' again." 403 print 404 return 1 405 return 0

406 407 DB_HELP = """ 408 The --db string is evaluated to build the DB object, which specifies 409 which database the buildmaster should use to hold scheduler state and 410 status information. The default (which creates an SQLite database in 411 BASEDIR/state.sqlite) is equivalent to: 412 413 --db='DBSpec("sqlite3", basedir+"/state.sqlite"))' 414 --db='sqlite:///state.sqlite' 415 416 To use a remote MySQL database instead, use something like: 417 418 --db='mysql://bbuser:bbpasswd@dbhost/bbdb' 419 """ 420

421 -class UpgradeMasterOptions(MakerBase):

422 optFlags = [ 423 ["replace", "r", "Replace any modified files without confirmation."], 424 ] 425 optParameters = [ 426 ["db", None, "sqlite:///state.sqlite", 427 "which DB to use for scheduler/status state. See below for syntax."], 428 ] 429

430 - def getSynopsis(self):

431 return "Usage: buildbot upgrade-master [options] [<basedir>]"

432 433 longdesc = """ 434 This command takes an existing buildmaster working directory and 435 adds/modifies the files there to work with the current version of 436 buildbot. When this command is finished, the buildmaster directory should 437 look much like a brand-new one created by the 'create-master' command. 438 439 Use this after you've upgraded your buildbot installation and before you 440 restart the buildmaster to use the new version. 441 442 If you have modified the files in your working directory, this command 443 will leave them untouched, but will put the new recommended contents in a 444 .new file (for example, if index.html has been modified, this command 445 will create index.html.new). You can then look at the new version and 446 decide how to merge its contents into your modified file. 447 """+DB_HELP+""" 448 When upgrading from a pre-0.8.0 release (which did not use a database), 449 this command will create the given database and migrate data from the old 450 pickle files into it, then move the pickle files out of the way (e.g. to 451 changes.pck.old). To revert to an older release, rename the pickle files 452 back. When you are satisfied with the new version, you can delete the old 453 pickle files. 454 """

455

456 -def upgradeMaster(config):

457 basedir = os.path.expanduser(config['basedir']) 458 m = Maker(config) 459 # TODO: check Makefile 460 # TODO: check TAC file 461 # check web files: index.html, default.css, robots.txt 462 m.upgrade_public_html({ 463 'bg_gradient.jpg' : util.sibpath(__file__, "../status/web/files/bg_gradient.jpg"), 464 'default.css' : util.sibpath(__file__, "../status/web/files/default.css"), 465 'robots.txt' : util.sibpath(__file__, "../status/web/files/robots.txt"), 466 'favicon.ico' : util.sibpath(__file__, "../status/web/files/favicon.ico"), 467 }) 468 m.populate_if_missing(os.path.join(basedir, "master.cfg.sample"), 469 util.sibpath(__file__, "sample.cfg"), 470 overwrite=True) 471 # if index.html exists, use it to override the root page tempalte 472 m.move_if_present(os.path.join(basedir, "public_html/index.html"), 473 os.path.join(basedir, "templates/root.html")) 474 475 from buildbot.db import dbspec 476 spec = dbspec.DBSpec.from_url(config["db"], basedir) 477 # TODO: check that TAC file specifies the right spec 478 479 # upgrade the db 480 from buildbot.db.schema import manager 481 sm = manager.DBSchemaManager(spec, basedir) 482 sm.upgrade() 483 484 # check the configuration 485 rc = m.check_master_cfg() 486 if rc: 487 return rc 488 if not config['quiet']: print "upgrade complete" 489 return 0

490 491

492 -class MasterOptions(MakerBase):

493 optFlags = [ 494 ["force", "f", 495 "Re-use an existing directory (will not overwrite master.cfg file)"], 496 ["relocatable", "r", 497 "Create a relocatable buildbot.tac"], 498 ["no-logrotate", "n", 499 "Do not permit buildmaster rotate logs by itself"] 500 ] 501 optParameters = [ 502 ["config", "c", "master.cfg", "name of the buildmaster config file"], 503 ["log-size", "s", "10000000", 504 "size at which to rotate twisted log files"], 505 ["log-count", "l", "10", 506 "limit the number of kept old twisted log files"], 507 ["db", None, "sqlite:///state.sqlite", 508 "which DB to use for scheduler/status state. See below for syntax."], 509 ]

510 - def getSynopsis(self):

511 return "Usage: buildbot create-master [options] [<basedir>]"

512 513 longdesc = """ 514 This command creates a buildmaster working directory and buildbot.tac file. 515 The master will live in <dir> and create various files there. If 516 --relocatable is given, then the resulting buildbot.tac file will be 517 written such that its containing directory is assumed to be the basedir. 518 This is generally a good idea. 519 520 At runtime, the master will read a configuration file (named 521 'master.cfg' by default) in its basedir. This file should contain python 522 code which eventually defines a dictionary named 'BuildmasterConfig'. 523 The elements of this dictionary are used to configure the Buildmaster. 524 See doc/config.xhtml for details about what can be controlled through 525 this interface. 526 """ + DB_HELP + """ 527 The --db string is stored verbatim in the buildbot.tac file, and 528 evaluated as 'buildbot start' time to pass a DBConnector instance into 529 the newly-created BuildMaster object. 530 """ 531

532 - def postOptions(self):

533 MakerBase.postOptions(self) 534 if not re.match('^\d+$', self['log-size']): 535 raise usage.UsageError("log-size parameter needs to be an int") 536 if not re.match('^\d+$', self['log-count']) and \ 537 self['log-count'] != 'None': 538 raise usage.UsageError("log-count parameter needs to be an int "+ 539 " or None")

540 541 542 masterTACTemplate = [""" 543 import os 544 545 from twisted.application import service 546 from buildbot.master import BuildMaster 547 548 basedir = r'%(basedir)s' 549 rotateLength = %(log-size)s 550 maxRotatedFiles = %(log-count)s 551 552 # if this is a relocatable tac file, get the directory containing the TAC 553 if basedir == '.': 554 import os.path 555 basedir = os.path.abspath(os.path.dirname(__file__)) 556 557 # note: this line is matched against to check that this is a buildmaster 558 # directory; do not edit it. 559 application = service.Application('buildmaster') 560 """, 561 """ 562 try: 563 from twisted.python.logfile import LogFile 564 from twisted.python.log import ILogObserver, FileLogObserver 565 logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength, 566 maxRotatedFiles=maxRotatedFiles) 567 application.setComponent(ILogObserver, FileLogObserver(logfile).emit) 568 except ImportError: 569 # probably not yet twisted 8.2.0 and beyond, can't set log yet 570 pass 571 """, 572 """ 573 configfile = r'%(config)s' 574 575 m = BuildMaster(basedir, configfile) 576 m.setServiceParent(application) 577 m.log_rotation.rotateLength = rotateLength 578 m.log_rotation.maxRotatedFiles = maxRotatedFiles 579 580 """] 581

582 -def createMaster(config):

583 m = Maker(config) 584 m.mkdir() 585 m.chdir() 586 if config['relocatable']: 587 config['basedir'] = '.' 588 if config['no-logrotate']: 589 masterTAC = "".join([masterTACTemplate[0]] + masterTACTemplate[2:]) 590 else: 591 masterTAC = "".join(masterTACTemplate) 592 contents = masterTAC % config 593 m.makeTAC(contents) 594 m.sampleconfig(util.sibpath(__file__, "sample.cfg")) 595 m.public_html({ 596 'bg_gradient.jpg' : util.sibpath(__file__, "../status/web/files/bg_gradient.jpg"), 597 'default.css' : util.sibpath(__file__, "../status/web/files/default.css"), 598 'robots.txt' : util.sibpath(__file__, "../status/web/files/robots.txt"), 599 'favicon.ico' : util.sibpath(__file__, "../status/web/files/favicon.ico"), 600 }) 601 m.makefile() 602 m.create_db() 603 604 if not m.quiet: print "buildmaster configured in %s" % m.basedir

605

606 -def stop(config, signame="TERM", wait=False):

607 import signal 608 basedir = config['basedir'] 609 quiet = config['quiet'] 610 611 if not isBuildmasterDir(config['basedir']): 612 print "not a buildmaster directory" 613 sys.exit(1) 614 615 os.chdir(basedir) 616 try: 617 f = open("twistd.pid", "rt") 618 except: 619 raise BuildbotNotRunningError 620 pid = int(f.read().strip()) 621 signum = getattr(signal, "SIG"+signame) 622 timer = 0 623 try: 624 os.kill(pid, signum) 625 except OSError, e: 626 if e.errno != 3: 627 raise 628 629 if not wait: 630 if not quiet: 631 print "sent SIG%s to process" % signame 632 return 633 time.sleep(0.1) 634 while timer < 10: 635 # poll once per second until twistd.pid goes away, up to 10 seconds 636 try: 637 os.kill(pid, 0) 638 except OSError: 639 if not quiet: 640 print "buildbot process %d is dead" % pid 641 return 642 timer += 1 643 time.sleep(1) 644 if not quiet: 645 print "never saw process go away"

646

647 -def restart(config):

648 basedir = config['basedir'] 649 quiet = config['quiet'] 650 651 if not isBuildmasterDir(basedir): 652 print "not a buildmaster directory" 653 sys.exit(1) 654 655 from buildbot.scripts.startup import start 656 try: 657 stop(config, wait=True) 658 except BuildbotNotRunningError: 659 pass 660 if not quiet: 661 print "now restarting buildbot process.." 662 start(config)

663 664

665 -class StartOptions(MakerBase):

666 optFlags = [ 667 ['quiet', 'q', "Don't display startup log messages"], 668 ]

669 - def getSynopsis(self):

670 return "Usage: buildbot start [<basedir>]"

671

672 -class StopOptions(MakerBase):

673 - def getSynopsis(self):

674 return "Usage: buildbot stop [<basedir>]"

675

676 -class ReconfigOptions(MakerBase):

677 optFlags = [ 678 ['quiet', 'q', "Don't display log messages about reconfiguration"], 679 ]

680 - def getSynopsis(self):

681 return "Usage: buildbot reconfig [<basedir>]"

682 683 684

685 -class RestartOptions(MakerBase):

686 optFlags = [ 687 ['quiet', 'q', "Don't display startup log messages"], 688 ]

689 - def getSynopsis(self):

690 return "Usage: buildbot restart [<basedir>]"

691

692 -class DebugClientOptions(OptionsWithOptionsFile):

693 optFlags = [ 694 ['help', 'h', "Display this message"], 695 ] 696 optParameters = [ 697 ["master", "m", None, 698 "Location of the buildmaster's slaveport (host:port)"], 699 ["passwd", "p", None, "Debug password to use"], 700 ] 701 buildbotOptions = [ 702 [ 'debugMaster', 'passwd' ], 703 [ 'master', 'master' ], 704 ]

705 - def getSynopsis(self):

706 return "Usage: buildbot debugclient [options]"

707

708 - def parseArgs(self, *args):

709 if len(args) > 0: 710 self['master'] = args[0] 711 if len(args) > 1: 712 self['passwd'] = args[1] 713 if len(args) > 2: 714 raise usage.UsageError("I wasn't expecting so many arguments")

715

716 -def debugclient(config):

717 from buildbot.clients import debug 718 719 master = config.get('master') 720 if master is None: 721 raise usage.UsageError("master must be specified: on the command " 722 "line or in ~/.buildbot/options") 723 724 passwd = config.get('passwd') 725 if passwd is None: 726 raise usage.UsageError("passwd must be specified: on the command " 727 "line or in ~/.buildbot/options") 728 729 d = debug.DebugWidget(master, passwd) 730 d.run()

731

732 -class StatusClientOptions(OptionsWithOptionsFile):

733 optFlags = [ 734 ['help', 'h', "Display this message"], 735 ] 736 optParameters = [ 737 ["master", "m", None, 738 "Location of the buildmaster's status port (host:port)"], 739 ["username", "u", "statusClient", "Username performing the trial build"], 740 ["passwd", None, "clientpw", "password for PB authentication"], 741 ] 742 buildbotOptions = [ 743 [ 'masterstatus', 'master' ], 744 ] 745

746 - def parseArgs(self, *args):

747 if len(args) > 0: 748 self['master'] = args[0] 749 if len(args) > 1: 750 raise usage.UsageError("I wasn't expecting so many arguments")

751

752 -class StatusLogOptions(StatusClientOptions):

753 - def getSynopsis(self):

754 return "Usage: buildbot statuslog [options]"

755

756 -class StatusGuiOptions(StatusClientOptions):

757 - def getSynopsis(self):

758 return "Usage: buildbot statusgui [options]"

759

760 -def statuslog(config):

761 from buildbot.clients import base 762 master = config.get('master') 763 if master is None: 764 raise usage.UsageError("master must be specified: on the command " 765 "line or in ~/.buildbot/options") 766 passwd = config.get('passwd') 767 username = config.get('username') 768 c = base.TextClient(master, username=username, passwd=passwd) 769 c.run()

770

771 -def statusgui(config):

772 from buildbot.clients import gtkPanes 773 master = config.get('master') 774 if master is None: 775 raise usage.UsageError("master must be specified: on the command " 776 "line or in ~/.buildbot/options") 777 c = gtkPanes.GtkClient(master) 778 c.run()

779

780 -class SendChangeOptions(OptionsWithOptionsFile):

781 - def __init__(self):

782 OptionsWithOptionsFile.__init__(self) 783 self['properties'] = {}

784 785 optParameters = [ 786 ("master", "m", None, 787 "Location of the buildmaster's PBListener (host:port)"), 788 # deprecated in 0.8.3; remove in 0.8.5 (bug #1711) 789 ("username", "u", None, "deprecated name for --who"), 790 ("auth", "a", None, "Authentication token - username:password, or prompt for password"), 791 ("who", "W", None, "Author of the commit"), 792 ("repository", "R", '', "Repository specifier"), 793 ("project", "P", '', "Project specifier"), 794 ("branch", "b", None, "Branch specifier"), 795 ("category", "C", None, "Category of repository"), 796 ("revision", "r", None, "Revision specifier"), 797 ("revision_file", None, None, "Filename containing revision spec"), 798 ("property", "p", None, 799 "A property for the change, in the format: name:value"), 800 ("comments", "c", None, "log message"), 801 ("logfile", "F", None, 802 "Read the log messages from this file (- for stdin)"), 803 ("when", "w", None, "timestamp to use as the change time"), 804 ("revlink", "l", '', "Revision link (revlink)"), 805 ] 806 807 buildbotOptions = [ 808 [ 'master', 'master' ], 809 [ 'who', 'who' ], 810 [ 'username', 'username' ], 811 [ 'branch', 'branch' ], 812 [ 'category', 'category' ], 813 ] 814

815 - def getSynopsis(self):

816 return "Usage: buildbot sendchange [options] filenames.."

817 - def parseArgs(self, *args):

818 self['files'] = args

819 - def opt_property(self, property):

820 name,value = property.split(':') 821 self['properties'][name] = value

822 823

824 -def sendchange(config, runReactor=False):

825 """Send a single change to the buildmaster's PBChangeSource. The 826 connection will be drpoped as soon as the Change has been sent.""" 827 from buildbot.clients.sendchange import Sender 828 829 who = config.get('who') 830 if not who and config.get('username'): 831 print "NOTE: --username/-u is deprecated: use --who/-W'" 832 who = config.get('username') 833 auth = config.get('auth') 834 master = config.get('master') 835 branch = config.get('branch') 836 category = config.get('category') 837 revision = config.get('revision') 838 properties = config.get('properties', {}) 839 repository = config.get('repository', '') 840 project = config.get('project', '') 841 revlink = config.get('revlink', '') 842 if config.get('when'): 843 when = float(config.get('when')) 844 else: 845 when = None 846 if config.get("revision_file"): 847 revision = open(config["revision_file"],"r").read() 848 849 comments = config.get('comments') 850 if not comments and config.get('logfile'): 851 if config['logfile'] == "-": 852 f = sys.stdin 853 else: 854 f = open(config['logfile'], "rt") 855 comments = f.read() 856 if comments is None: 857 comments = "" 858 859 files = config.get('files', []) 860 861 # fix up the auth with a password if none was given 862 if not auth: 863 auth = 'change:changepw' 864 if ':' not in auth: 865 import getpass 866 pw = getpass.getpass("Enter password for '%s': " % auth) 867 auth = "%s:%s" % (auth, pw) 868 auth = auth.split(':', 1) 869 870 assert who, "you must provide a committer (--who)" 871 assert master, "you must provide the master location" 872 873 s = Sender(master, auth) 874 d = s.send(branch, revision, comments, files, who=who, category=category, when=when, 875 properties=properties, repository=repository, project=project, 876 revlink=revlink) 877 if runReactor: 878 status = [True] 879 def failed(res): 880 status[0] = False 881 s.printFailure(res)

882 d.addCallbacks(s.printSuccess, failed) 883 d.addBoth(s.stop) 884 s.run() 885 return status[0] 886 return d 887 888

889 -class ForceOptions(OptionsWithOptionsFile):

890 optParameters = [ 891 ["builder", None, None, "which Builder to start"], 892 ["branch", None, None, "which branch to build"], 893 ["revision", None, None, "which revision to build"], 894 ["reason", None, None, "the reason for starting the build"], 895 ] 896

897 - def parseArgs(self, *args):

898 args = list(args) 899 if len(args) > 0: 900 if self['builder'] is not None: 901 raise usage.UsageError("--builder provided in two ways") 902 self['builder'] = args.pop(0) 903 if len(args) > 0: 904 if self['reason'] is not None: 905 raise usage.UsageError("--reason provided in two ways") 906 self['reason'] = " ".join(args)

907 908

909 -class TryOptions(OptionsWithOptionsFile):

910 optParameters = [ 911 ["connect", "c", None, 912 "how to reach the buildmaster, either 'ssh' or 'pb'"], 913 # for ssh, use --tryhost, --username, and --trydir 914 ["tryhost", None, None, 915 "the hostname (used by ssh) for the buildmaster"], 916 ["trydir", None, None, 917 "the directory (on the tryhost) where tryjobs are deposited"], 918 ["username", "u", None, "Username performing the trial build"], 919 # for PB, use --master, --username, and --passwd 920 ["master", "m", None, 921 "Location of the buildmaster's PBListener (host:port)"], 922 ["passwd", None, None, "password for PB authentication"], 923 924 ["diff", None, None, 925 "Filename of a patch to use instead of scanning a local tree. Use '-' for stdin."], 926 ["patchlevel", "p", 0, 927 "Number of slashes to remove from patch pathnames, like the -p option to 'patch'"], 928 929 ["baserev", None, None, 930 "Base revision to use instead of scanning a local tree."], 931 932 ["vc", None, None, 933 "The VC system in use, one of: cvs,svn,bzr,darcs,p4"], 934 ["branch", None, None, 935 "The branch in use, for VC systems that can't figure it out" 936 " themselves"], 937 938 ["builder", "b", None, 939 "Run the trial build on this Builder. Can be used multiple times."], 940 ["properties", None, None, 941 "A set of properties made available in the build environment, format:prop1=value1,prop2=value2..."], 942 943 ["try-topfile", None, None, 944 "Name of a file at the top of the tree, used to find the top. Only needed for SVN and CVS."], 945 ["try-topdir", None, None, 946 "Path to the top of the working copy. Only needed for SVN and CVS."], 947 948 ] 949 950 optFlags = [ 951 ["wait", None, "wait until the builds have finished"], 952 ["dryrun", 'n', "Gather info, but don't actually submit."], 953 ["get-builder-names", None, "Get the names of available builders. Doesn't submit anything. Only supported for 'pb' connections."], 954 ] 955 956 # here it is, the definitive, quirky mapping of .buildbot/options names to 957 # command-line options. Design by committee, anyone? 958 buildbotOptions = [ 959 [ 'try_connect', 'connect' ], 960 #[ 'try_builders', 'builders' ], <-- handled in postOptions 961 [ 'try_vc', 'vc' ], 962 [ 'try_branch', 'branch' ], 963 [ 'try_topdir', 'try-topdir' ], 964 [ 'try_topfile', 'try-topfile' ], 965 [ 'try_host', 'tryhost' ], 966 [ 'try_username', 'username' ], 967 [ 'try_dir', 'trydir' ], 968 [ 'try_password', 'passwd' ], 969 [ 'try_master', 'master' ], 970 #[ 'try_wait', 'wait' ], <-- handled in postOptions 971 [ 'masterstatus', 'master' ], 972 ] 973

974 - def __init__(self):

975 OptionsWithOptionsFile.__init__(self) 976 self['builders'] = [] 977 self['properties'] = {}

978

979 - def opt_builder(self, option):

980 self['builders'].append(option)

981

982 - def opt_properties(self, option):

983 # We need to split the value of this option into a dictionary of properties 984 properties = {} 985 propertylist = option.split(",") 986 for i in range(0,len(propertylist)): 987 print propertylist[i] 988 splitproperty = propertylist[i].split("=") 989 properties[splitproperty[0]] = splitproperty[1] 990 self['properties'] = properties

991

992 - def opt_patchlevel(self, option):

993 self['patchlevel'] = int(option)

994

995 - def getSynopsis(self):

996 return "Usage: buildbot try [options]"

997

998 - def postOptions(self):

999 opts = loadOptionsFile() 1000 if not self['builders']: 1001 self['builders'] = opts.get('try_builders', []) 1002 if opts.get('try_wait', False): 1003 self['wait'] = True

1004

1005 -def doTry(config):

1006 from buildbot.clients import tryclient 1007 t = tryclient.Try(config) 1008 t.run()

1009

1010 -class TryServerOptions(OptionsWithOptionsFile):

1011 optParameters = [ 1012 ["jobdir", None, None, "the jobdir (maildir) for submitting jobs"], 1013 ]

1014 - def getSynopsis(self):

1015 return "Usage: buildbot tryserver [options]"

1016 1017

1018 -def doTryServer(config):

1019 try: 1020 from hashlib import md5 1021 assert md5 1022 except ImportError: 1023 # For Python 2.4 compatibility 1024 import md5 1025 jobdir = os.path.expanduser(config["jobdir"]) 1026 job = sys.stdin.read() 1027 # now do a 'safecat'-style write to jobdir/tmp, then move atomically to 1028 # jobdir/new . Rather than come up with a unique name randomly, I'm just 1029 # going to MD5 the contents and prepend a timestamp. 1030 timestring = "%d" % time.time() 1031 try: 1032 m = md5() 1033 except TypeError: 1034 # For Python 2.4 compatibility 1035 m = md5.new() 1036 m.update(job) 1037 jobhash = m.hexdigest() 1038 fn = "%s-%s" % (timestring, jobhash) 1039 tmpfile = os.path.join(jobdir, "tmp", fn) 1040 newfile = os.path.join(jobdir, "new", fn) 1041 f = open(tmpfile, "w") 1042 f.write(job) 1043 f.close() 1044 os.rename(tmpfile, newfile)

1045 1046

1047 -class CheckConfigOptions(OptionsWithOptionsFile):

1048 optFlags = [ 1049 ['quiet', 'q', "Don't display error messages or tracebacks"], 1050 ] 1051

1052 - def getSynopsis(self):

1053 return "Usage: buildbot checkconfig [configFile]\n" + \ 1054 " If not specified, 'master.cfg' will be used as 'configFile'"

1055

1056 - def parseArgs(self, *args):

1057 if len(args) >= 1: 1058 self['configFile'] = args[0] 1059 else: 1060 self['configFile'] = 'master.cfg'

1061 1062

1063 -def doCheckConfig(config):

1064 quiet = config.get('quiet') 1065 configFileName = config.get('configFile') 1066 try: 1067 from buildbot.scripts.checkconfig import ConfigLoader 1068 if os.path.isdir(configFileName): 1069 ConfigLoader(basedir=configFileName) 1070 else: 1071 ConfigLoader(configFileName=configFileName) 1072 except: 1073 if not quiet: 1074 # Print out the traceback in a nice format 1075 t, v, tb = sys.exc_info() 1076 traceback.print_exception(t, v, tb) 1077 sys.exit(1) 1078 1079 if not quiet: 1080 print "Config file is good!"

1081 1082

1083 -class Options(usage.Options):

1084 synopsis = "Usage: buildbot <command> [command options]" 1085 1086 subCommands = [ 1087 # the following are all admin commands 1088 ['create-master', None, MasterOptions, 1089 "Create and populate a directory for a new buildmaster"], 1090 ['upgrade-master', None, UpgradeMasterOptions, 1091 "Upgrade an existing buildmaster directory for the current version"], 1092 ['start', None, StartOptions, "Start a buildmaster"], 1093 ['stop', None, StopOptions, "Stop a buildmaster"], 1094 ['restart', None, RestartOptions, 1095 "Restart a buildmaster"], 1096 1097 ['reconfig', None, ReconfigOptions, 1098 "SIGHUP a buildmaster to make it re-read the config file"], 1099 ['sighup', None, ReconfigOptions, 1100 "SIGHUP a buildmaster to make it re-read the config file"], 1101 1102 ['sendchange', None, SendChangeOptions, 1103 "Send a change to the buildmaster"], 1104 1105 ['debugclient', None, DebugClientOptions, 1106 "Launch a small debug panel GUI"], 1107 1108 ['statuslog', None, StatusLogOptions, 1109 "Emit current builder status to stdout"], 1110 ['statusgui', None, StatusGuiOptions, 1111 "Display a small window showing current builder status"], 1112 1113 #['force', None, ForceOptions, "Run a build"], 1114 ['try', None, TryOptions, "Run a build with your local changes"], 1115 1116 ['tryserver', None, TryServerOptions, 1117 "buildmaster-side 'try' support function, not for users"], 1118 1119 ['checkconfig', None, CheckConfigOptions, 1120 "test the validity of a master.cfg config file"], 1121 1122 # TODO: 'watch' 1123 ] 1124

1125 - def opt_version(self):

1126 import buildbot 1127 print "Buildbot version: %s" % buildbot.version 1128 usage.Options.opt_version(self)

1129

1130 - def opt_verbose(self):

1131 from twisted.python import log 1132 log.startLogging(sys.stderr)

1133

1134 - def postOptions(self):

1135 if not hasattr(self, 'subOptions'): 1136 raise usage.UsageError("must specify a command")

1137 1138

1139 -def run():

1140 config = Options() 1141 try: 1142 config.parseOptions() 1143 except usage.error, e: 1144 print "%s: %s" % (sys.argv[0], e) 1145 print 1146 c = getattr(config, 'subOptions', config) 1147 print str(c) 1148 sys.exit(1) 1149 1150 command = config.subCommand 1151 so = config.subOptions 1152 1153 if command == "create-master": 1154 createMaster(so) 1155 elif command == "upgrade-master": 1156 upgradeMaster(so) 1157 elif command == "start": 1158 from buildbot.scripts.startup import start 1159 1160 if not isBuildmasterDir(so['basedir']): 1161 print "not a buildmaster directory" 1162 sys.exit(1) 1163 1164 start(so) 1165 elif command == "stop": 1166 try: 1167 stop(so, wait=True) 1168 except BuildbotNotRunningError: 1169 if not so['quiet']: 1170 print "buildmaster not running" 1171 sys.exit(0) 1172 1173 elif command == "restart": 1174 restart(so) 1175 elif command == "reconfig" or command == "sighup": 1176 from buildbot.scripts.reconfig import Reconfigurator 1177 Reconfigurator().run(so) 1178 elif command == "sendchange": 1179 if not sendchange(so, True): 1180 sys.exit(1) 1181 elif command == "debugclient": 1182 debugclient(so) 1183 elif command == "statuslog": 1184 statuslog(so) 1185 elif command == "statusgui": 1186 statusgui(so) 1187 elif command == "try": 1188 doTry(so) 1189 elif command == "tryserver": 1190 doTryServer(so) 1191 elif command == "checkconfig": 1192 doCheckConfig(so) 1193 sys.exit(0)

1194

Source Code for Module buildbot.scripts.runner