buildbot.scripts.runner

1 # -*- test-case-name: buildbot.test.test_runner -*- 2 3 # N.B.: don't import anything that might pull in a reactor yet. Some of our 4 # subcommands want to load modules that need the gtk reactor. 5 import os, sys, stat, re, time 6 import traceback 7 from twisted.python import usage, util, runtime 8 9 from buildbot.interfaces import BuildbotNotRunningError 10 11 # the create/start/stop commands should all be run as the same user, 12 # preferably a separate 'buildbot' account. 13 14 # Note that the terms 'options' and 'config' are used intechangeably here - in 15 # fact, they are intercanged several times. Caveat legator. 16

17 -class OptionsWithOptionsFile(usage.Options):

18 # subclasses should set this to a list-of-lists in order to source the 19 # .buildbot/options file. 20 # buildbotOptions = [ [ 'optfile-name', 'option-name' ], .. ] 21 buildbotOptions = None 22

23 - def __init__(self, *args):

24 # for options in self.buildbotOptions, optParameters, and the options 25 # file, change the default in optParameters *before* calling through 26 # to the parent constructor 27 28 if self.buildbotOptions: 29 optfile = loadOptionsFile() 30 for optfile_name, option_name in self.buildbotOptions: 31 for i in range(len(self.optParameters)): 32 if self.optParameters[i][0] == option_name and optfile_name in optfile: 33 self.optParameters[i][2] = optfile[optfile_name] 34 usage.Options.__init__(self, *args)

35

36 -def loadOptionsFile(filename="options", here=None, home=None):

37 """Find the .buildbot/FILENAME file. Crawl from the current directory up 38 towards the root, and also look in ~/.buildbot . The first directory 39 that's owned by the user and has the file we're looking for wins. Windows 40 skips the owned-by-user test. 41 42 @rtype: dict 43 @return: a dictionary of names defined in the options file. If no options 44 file was found, return an empty dict. 45 """ 46 47 if here is None: 48 here = os.getcwd() 49 here = os.path.abspath(here) 50 51 if home is None: 52 if runtime.platformType == 'win32': 53 # never trust env-vars, use the proper API 54 from win32com.shell import shellcon, shell 55 appdata = shell.SHGetFolderPath(0, shellcon.CSIDL_APPDATA, 0, 0) 56 home = os.path.join(appdata, "buildbot") 57 else: 58 home = os.path.expanduser("~/.buildbot") 59 60 searchpath = [] 61 toomany = 20 62 while True: 63 searchpath.append(os.path.join(here, ".buildbot")) 64 next = os.path.dirname(here) 65 if next == here: 66 break # we've hit the root 67 here = next 68 toomany -= 1 # just in case 69 if toomany == 0: 70 raise ValueError("Hey, I seem to have wandered up into the " 71 "infinite glories of the heavens. Oops.") 72 searchpath.append(home) 73 74 localDict = {} 75 76 for d in searchpath: 77 if os.path.isdir(d): 78 if runtime.platformType != 'win32': 79 if os.stat(d)[stat.ST_UID] != os.getuid(): 80 print "skipping %s because you don't own it" % d 81 continue # security, skip other people's directories 82 optfile = os.path.join(d, filename) 83 if os.path.exists(optfile): 84 try: 85 f = open(optfile, "r") 86 options = f.read() 87 exec options in localDict 88 except: 89 print "error while reading %s" % optfile 90 raise 91 break 92 93 for k in localDict.keys(): 94 if k.startswith("__"): 95 del localDict[k] 96 return localDict

97

98 -class MakerBase(OptionsWithOptionsFile):

99 optFlags = [ 100 ['help', 'h', "Display this message"], 101 ["quiet", "q", "Do not emit the commands being run"], 102 ] 103 104 longdesc = """ 105 Operates upon the specified <basedir> (or the current directory, if not 106 specified). 107 """ 108 109 opt_h = usage.Options.opt_help 110

111 - def parseArgs(self, *args):

112 if len(args) > 0: 113 self['basedir'] = args[0] 114 else: 115 # Use the current directory if no basedir was specified. 116 self['basedir'] = os.getcwd() 117 if len(args) > 1: 118 raise usage.UsageError("I wasn't expecting so many arguments")

119

120 - def postOptions(self):

121 self['basedir'] = os.path.abspath(self['basedir'])

122 123 makefile_sample = """# -*- makefile -*- 124 125 # This is a simple makefile which lives in a buildmaster/buildslave 126 # directory (next to the buildbot.tac file). It allows you to start/stop the 127 # master or slave by doing 'make start' or 'make stop'. 128 129 # The 'reconfig' target will tell a buildmaster to reload its config file. 130 131 start: 132 twistd --no_save -y buildbot.tac 133 134 stop: 135 if [ -e twistd.pid ]; \\ 136 then kill `cat twistd.pid`; \\ 137 else echo "Nothing to stop."; \\ 138 fi 139 140 reconfig: 141 if [ -e twistd.pid ]; \\ 142 then kill -HUP `cat twistd.pid`; \\ 143 else echo "Nothing to reconfig."; \\ 144 fi 145 146 log: 147 if [ -e twistd.log ]; \\ 148 then tail -f twistd.log; \\ 149 else echo "Nothing to tail."; \\ 150 fi 151 """ 152

153 -class Maker:

154 - def __init__(self, config):

155 self.config = config 156 self.basedir = config['basedir'] 157 self.force = config.get('force', False) 158 self.quiet = config['quiet']

159

160 - def mkdir(self):

161 if os.path.exists(self.basedir): 162 if not self.quiet: 163 print "updating existing installation" 164 return 165 if not self.quiet: print "mkdir", self.basedir 166 os.mkdir(self.basedir)

167

168 - def mkinfo(self):

169 path = os.path.join(self.basedir, "info") 170 if not os.path.exists(path): 171 if not self.quiet: print "mkdir", path 172 os.mkdir(path) 173 created = False 174 admin = os.path.join(path, "admin") 175 if not os.path.exists(admin): 176 if not self.quiet: 177 print "Creating info/admin, you need to edit it appropriately" 178 f = open(admin, "wt") 179 f.write("Your Name Here <admin@youraddress.invalid>\n") 180 f.close() 181 created = True 182 host = os.path.join(path, "host") 183 if not os.path.exists(host): 184 if not self.quiet: 185 print "Creating info/host, you need to edit it appropriately" 186 f = open(host, "wt") 187 f.write("Please put a description of this build host here\n") 188 f.close() 189 created = True 190 access_uri = os.path.join(path, "access_uri") 191 if not os.path.exists(access_uri): 192 if not self.quiet: 193 print "Not creating info/access_uri - add it if you wish" 194 if created and not self.quiet: 195 print "Please edit the files in %s appropriately." % path

196

197 - def chdir(self):

198 if not self.quiet: print "chdir", self.basedir 199 os.chdir(self.basedir)

200

201 - def makeTAC(self, contents, secret=False):

202 tacfile = "buildbot.tac" 203 if os.path.exists(tacfile): 204 oldcontents = open(tacfile, "rt").read() 205 if oldcontents == contents: 206 if not self.quiet: 207 print "buildbot.tac already exists and is correct" 208 return 209 if not self.quiet: 210 print "not touching existing buildbot.tac" 211 print "creating buildbot.tac.new instead" 212 tacfile = "buildbot.tac.new" 213 f = open(tacfile, "wt") 214 f.write(contents) 215 f.close() 216 if secret: 217 os.chmod(tacfile, 0600)

218

219 - def makefile(self):

220 target = "Makefile.sample" 221 if os.path.exists(target): 222 oldcontents = open(target, "rt").read() 223 if oldcontents == makefile_sample: 224 if not self.quiet: 225 print "Makefile.sample already exists and is correct" 226 return 227 if not self.quiet: 228 print "replacing Makefile.sample" 229 else: 230 if not self.quiet: 231 print "creating Makefile.sample" 232 f = open(target, "wt") 233 f.write(makefile_sample) 234 f.close()

235

236 - def sampleconfig(self, source):

237 target = "master.cfg.sample" 238 config_sample = open(source, "rt").read() 239 if os.path.exists(target): 240 oldcontents = open(target, "rt").read() 241 if oldcontents == config_sample: 242 if not self.quiet: 243 print "master.cfg.sample already exists and is up-to-date" 244 return 245 if not self.quiet: 246 print "replacing master.cfg.sample" 247 else: 248 if not self.quiet: 249 print "creating master.cfg.sample" 250 f = open(target, "wt") 251 f.write(config_sample) 252 f.close() 253 os.chmod(target, 0600)

254

255 - def public_html(self, files):

256 webdir = os.path.join(self.basedir, "public_html") 257 if os.path.exists(webdir): 258 if not self.quiet: 259 print "public_html/ already exists: not replacing" 260 return 261 else: 262 os.mkdir(webdir) 263 if not self.quiet: 264 print "populating public_html/" 265 for target, source in files.iteritems(): 266 target = os.path.join(webdir, target) 267 f = open(target, "wt") 268 f.write(open(source, "rt").read()) 269 f.close()

270

271 - def create_db(self):

272 from buildbot.db import dbspec, exceptions, schema 273 spec = dbspec.DBSpec.from_url(self.config["db"], self.basedir) 274 if not self.config['quiet']: print "creating database" 275 276 # upgrade from "nothing" 277 sm = schema.DBSchemaManager(spec, self.basedir) 278 if sm.get_db_version() != 0: 279 raise exceptions.DBAlreadyExistsError 280 sm.upgrade()

281

282 - def populate_if_missing(self, target, source, overwrite=False):

283 new_contents = open(source, "rt").read() 284 if os.path.exists(target): 285 old_contents = open(target, "rt").read() 286 if old_contents != new_contents: 287 if overwrite: 288 if not self.quiet: 289 print "%s has old/modified contents" % target 290 print " overwriting it with new contents" 291 open(target, "wt").write(new_contents) 292 else: 293 if not self.quiet: 294 print "%s has old/modified contents" % target 295 print " writing new contents to %s.new" % target 296 open(target + ".new", "wt").write(new_contents) 297 # otherwise, it's up to date 298 else: 299 if not self.quiet: 300 print "populating %s" % target 301 open(target, "wt").write(new_contents)

302

303 - def move_if_present(self, source, dest):

304 if os.path.exists(source): 305 if os.path.exists(dest): 306 print "Notice: %s now overrides %s" % (dest, source) 307 print " as the latter is not used by buildbot anymore." 308 print " Decide which one you want to keep." 309 else: 310 try: 311 print "Notice: Moving %s to %s." % (source, dest) 312 print " You can (and probably want to) remove it if you haven't modified this file." 313 os.renames(source, dest) 314 except Exception, e: 315 print "Error moving %s to %s: %s" % (source, dest, str(e))

316

317 - def upgrade_public_html(self, files):

318 webdir = os.path.join(self.basedir, "public_html") 319 if not os.path.exists(webdir): 320 if not self.quiet: 321 print "populating public_html/" 322 os.mkdir(webdir) 323 for target, source in files.iteritems(): 324 self.populate_if_missing(os.path.join(webdir, target), 325 source)

326

327 - def check_master_cfg(self):

328 from buildbot.master import BuildMaster 329 from twisted.python import log, failure 330 331 master_cfg = os.path.join(self.basedir, "master.cfg") 332 if not os.path.exists(master_cfg): 333 if not self.quiet: 334 print "No master.cfg found" 335 return 1 336 337 # side-effects of loading the config file: 338 339 # for each Builder defined in c['builders'], if the status directory 340 # didn't already exist, it will be created, and the 341 # $BUILDERNAME/builder pickle might be created (with a single 342 # "builder created" event). 343 344 # we put basedir in front of sys.path, because that's how the 345 # buildmaster itself will run, and it is quite common to have the 346 # buildmaster import helper classes from other .py files in its 347 # basedir. 348 349 if sys.path[0] != self.basedir: 350 sys.path.insert(0, self.basedir) 351 352 m = BuildMaster(self.basedir) 353 # we need to route log.msg to stdout, so any problems can be seen 354 # there. But if everything goes well, I'd rather not clutter stdout 355 # with log messages. So instead we add a logObserver which gathers 356 # messages and only displays them if something goes wrong. 357 messages = [] 358 log.addObserver(messages.append) 359 try: 360 # this will raise an exception if there's something wrong with 361 # the config file. Note that this BuildMaster instance is never 362 # started, so it won't actually do anything with the 363 # configuration. 364 m.loadConfig(open(master_cfg, "r"), check_synchronously_only=True) 365 except: 366 f = failure.Failure() 367 if not self.quiet: 368 print 369 for m in messages: 370 print "".join(m['message']) 371 print f 372 print 373 print "An error was detected in the master.cfg file." 374 print "Please correct the problem and run 'buildbot upgrade-master' again." 375 print 376 return 1 377 return 0

378 379 DB_HELP = """ 380 The --db string is evaluated to build the DB object, which specifies 381 which database the buildmaster should use to hold scheduler state and 382 status information. The default (which creates an SQLite database in 383 BASEDIR/state.sqlite) is equivalent to: 384 385 --db='DBSpec("sqlite3", basedir+"/state.sqlite"))' 386 --db='sqlite:///state.sqlite' 387 388 To use a remote MySQL database instead, use something like: 389 390 --db='mysql://bbuser:bbpasswd@dbhost/bbdb' 391 """ 392

393 -class UpgradeMasterOptions(MakerBase):

394 optFlags = [ 395 ["replace", "r", "Replace any modified files without confirmation."], 396 ] 397 optParameters = [ 398 ["db", None, "sqlite:///state.sqlite", 399 "which DB to use for scheduler/status state. See below for syntax."], 400 ] 401

402 - def getSynopsis(self):

403 return "Usage: buildbot upgrade-master [options] [<basedir>]"

404 405 longdesc = """ 406 This command takes an existing buildmaster working directory and 407 adds/modifies the files there to work with the current version of 408 buildbot. When this command is finished, the buildmaster directory should 409 look much like a brand-new one created by the 'create-master' command. 410 411 Use this after you've upgraded your buildbot installation and before you 412 restart the buildmaster to use the new version. 413 414 If you have modified the files in your working directory, this command 415 will leave them untouched, but will put the new recommended contents in a 416 .new file (for example, if index.html has been modified, this command 417 will create index.html.new). You can then look at the new version and 418 decide how to merge its contents into your modified file. 419 """+DB_HELP+""" 420 When upgrading from a pre-0.8.0 release (which did not use a database), 421 this command will create the given database and migrate data from the old 422 pickle files into it, then move the pickle files out of the way (e.g. to 423 changes.pck.old). To revert to an older release, rename the pickle files 424 back. When you are satisfied with the new version, you can delete the old 425 pickle files. 426 """

427

428 -def upgradeMaster(config):

429 basedir = os.path.expanduser(config['basedir']) 430 m = Maker(config) 431 # TODO: check Makefile 432 # TODO: check TAC file 433 # check web files: index.html, default.css, robots.txt 434 m.upgrade_public_html({ 435 'bg_gradient.jpg' : util.sibpath(__file__, "../status/web/files/bg_gradient.jpg"), 436 'default.css' : util.sibpath(__file__, "../status/web/files/default.css"), 437 'robots.txt' : util.sibpath(__file__, "../status/web/files/robots.txt"), 438 'favicon.ico' : util.sibpath(__file__, "../status/web/files/favicon.ico"), 439 }) 440 m.populate_if_missing(os.path.join(basedir, "master.cfg.sample"), 441 util.sibpath(__file__, "sample.cfg"), 442 overwrite=True) 443 # if index.html exists, use it to override the root page tempalte 444 m.move_if_present(os.path.join(basedir, "public_html/index.html"), 445 os.path.join(basedir, "templates/root.html")) 446 447 from buildbot.db import schema, connector, dbspec 448 spec = dbspec.DBSpec.from_url(config["db"], basedir) 449 # TODO: check that TAC file specifies the right spec 450 451 # upgrade the db 452 sm = schema.DBSchemaManager(spec, basedir) 453 sm.upgrade() 454 455 db = connector.DBConnector(spec) 456 db.start() 457 458 rc = m.check_master_cfg() 459 if rc: 460 return rc 461 if not config['quiet']: print "upgrade complete" 462 return 0

463 464

465 -class MasterOptions(MakerBase):

466 optFlags = [ 467 ["force", "f", 468 "Re-use an existing directory (will not overwrite master.cfg file)"], 469 ["relocatable", "r", 470 "Create a relocatable buildbot.tac"], 471 ] 472 optParameters = [ 473 ["config", "c", "master.cfg", "name of the buildmaster config file"], 474 ["log-size", "s", "1000000", 475 "size at which to rotate twisted log files"], 476 ["log-count", "l", "None", 477 "limit the number of kept old twisted log files"], 478 ["db", None, "sqlite:///state.sqlite", 479 "which DB to use for scheduler/status state. See below for syntax."], 480 ]

481 - def getSynopsis(self):

482 return "Usage: buildbot create-master [options] [<basedir>]"

483 484 longdesc = """ 485 This command creates a buildmaster working directory and buildbot.tac file. 486 The master will live in <dir> and create various files there. If 487 --relocatable is given, then the resulting buildbot.tac file will be 488 written such that its containing directory is assumed to be the basedir. 489 This is generally a good idea. 490 491 At runtime, the master will read a configuration file (named 492 'master.cfg' by default) in its basedir. This file should contain python 493 code which eventually defines a dictionary named 'BuildmasterConfig'. 494 The elements of this dictionary are used to configure the Buildmaster. 495 See doc/config.xhtml for details about what can be controlled through 496 this interface. 497 """ + DB_HELP + """ 498 The --db string is stored verbatim in the buildbot.tac file, and 499 evaluated as 'buildbot start' time to pass a DBConnector instance into 500 the newly-created BuildMaster object. 501 """ 502

503 - def postOptions(self):

504 MakerBase.postOptions(self) 505 if not re.match('^\d+$', self['log-size']): 506 raise usage.UsageError("log-size parameter needs to be an int") 507 if not re.match('^\d+$', self['log-count']) and \ 508 self['log-count'] != 'None': 509 raise usage.UsageError("log-count parameter needs to be an int "+ 510 " or None")

511 512 513 masterTAC = """ 514 import os 515 516 from twisted.application import service 517 from buildbot.master import BuildMaster 518 519 basedir = r'%(basedir)s' 520 rotateLength = %(log-size)s 521 maxRotatedFiles = %(log-count)s 522 523 # if this is a relocatable tac file, get the directory containing the TAC 524 if basedir == '.': 525 import os.path 526 basedir = os.path.abspath(os.path.dirname(__file__)) 527 528 application = service.Application('buildmaster') 529 try: 530 from twisted.python.logfile import LogFile 531 from twisted.python.log import ILogObserver, FileLogObserver 532 logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength, 533 maxRotatedFiles=maxRotatedFiles) 534 application.setComponent(ILogObserver, FileLogObserver(logfile).emit) 535 except ImportError: 536 # probably not yet twisted 8.2.0 and beyond, can't set log yet 537 pass 538 539 configfile = r'%(config)s' 540 541 m = BuildMaster(basedir, configfile) 542 m.setServiceParent(application) 543 m.log_rotation.rotateLength = rotateLength 544 m.log_rotation.maxRotatedFiles = maxRotatedFiles 545 546 """ 547

548 -def createMaster(config):

549 m = Maker(config) 550 m.mkdir() 551 m.chdir() 552 if config['relocatable']: 553 config['basedir'] = '.' 554 contents = masterTAC % config 555 m.makeTAC(contents) 556 m.sampleconfig(util.sibpath(__file__, "sample.cfg")) 557 m.public_html({ 558 'bg_gradient.jpg' : util.sibpath(__file__, "../status/web/files/bg_gradient.jpg"), 559 'default.css' : util.sibpath(__file__, "../status/web/files/default.css"), 560 'robots.txt' : util.sibpath(__file__, "../status/web/files/robots.txt"), 561 'favicon.ico' : util.sibpath(__file__, "../status/web/files/favicon.ico"), 562 }) 563 m.makefile() 564 m.create_db() 565 566 if not m.quiet: print "buildmaster configured in %s" % m.basedir

567

568 -class SlaveOptions(MakerBase):

569 optFlags = [ 570 ["force", "f", "Re-use an existing directory"], 571 ["relocatable", "r", 572 "Create a relocatable buildbot.tac"], 573 ] 574 optParameters = [ 575 # ["name", "n", None, "Name for this build slave"], 576 # ["passwd", "p", None, "Password for this build slave"], 577 # ["basedir", "d", ".", "Base directory to use"], 578 # ["master", "m", "localhost:8007", 579 # "Location of the buildmaster (host:port)"], 580 581 ["keepalive", "k", 600, 582 "Interval at which keepalives should be sent (in seconds)"], 583 ["usepty", None, 0, 584 "(1 or 0) child processes should be run in a pty (default 0)"], 585 ["umask", None, "None", 586 "controls permissions of generated files. Use --umask=022 to be world-readable"], 587 ["maxdelay", None, 300, 588 "Maximum time between connection attempts"], 589 ["log-size", "s", "1000000", 590 "size at which to rotate twisted log files"], 591 ["log-count", "l", "None", 592 "limit the number of kept old twisted log files"], 593 ] 594 595 longdesc = """ 596 This command creates a buildslave working directory and buildbot.tac 597 file. The bot will use the <name> and <passwd> arguments to authenticate 598 itself when connecting to the master. All commands are run in a 599 build-specific subdirectory of <basedir>. <master> is a string of the 600 form 'hostname:port', and specifies where the buildmaster can be reached. 601 602 <name>, <passwd>, and <master> will be provided by the buildmaster 603 administrator for your bot. You must choose <basedir> yourself. 604 """ 605

606 - def getSynopsis(self):

607 return "Usage: buildbot create-slave [options] <basedir> <master> <name> <passwd>"

608

609 - def parseArgs(self, *args):

610 if len(args) < 4: 611 raise usage.UsageError("command needs more arguments") 612 basedir, master, name, passwd = args 613 if master[:5] == "http:": 614 raise usage.UsageError("<master> is not a URL - do not use URL") 615 self['basedir'] = basedir 616 self['master'] = master 617 self['name'] = name 618 self['passwd'] = passwd

619

620 - def postOptions(self):

621 MakerBase.postOptions(self) 622 self['usepty'] = int(self['usepty']) 623 self['keepalive'] = int(self['keepalive']) 624 self['maxdelay'] = int(self['maxdelay']) 625 if self['master'].find(":") == -1: 626 raise usage.UsageError("--master must be in the form host:portnum") 627 if not re.match('^\d+$', self['log-size']): 628 raise usage.UsageError("log-size parameter needs to be an int") 629 if not re.match('^\d+$', self['log-count']) and \ 630 self['log-count'] != 'None': 631 raise usage.UsageError("log-count parameter needs to be an int "+ 632 " or None")

633 634 slaveTAC = """ 635 import os 636 637 from twisted.application import service 638 from buildbot.slave.bot import BuildSlave 639 640 basedir = r'%(basedir)s' 641 rotateLength = %(log-size)s 642 maxRotatedFiles = %(log-count)s 643 644 # if this is a relocatable tac file, get the directory containing the TAC 645 if basedir == '.': 646 import os.path 647 basedir = os.path.abspath(os.path.dirname(__file__)) 648 649 application = service.Application('buildslave') 650 try: 651 from twisted.python.logfile import LogFile 652 from twisted.python.log import ILogObserver, FileLogObserver 653 logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength, 654 maxRotatedFiles=maxRotatedFiles) 655 application.setComponent(ILogObserver, FileLogObserver(logfile).emit) 656 except ImportError: 657 # probably not yet twisted 8.2.0 and beyond, can't set log yet 658 pass 659 660 buildmaster_host = '%(host)s' 661 port = %(port)d 662 slavename = '%(name)s' 663 passwd = '%(passwd)s' 664 keepalive = %(keepalive)d 665 usepty = %(usepty)d 666 umask = %(umask)s 667 maxdelay = %(maxdelay)d 668 669 s = BuildSlave(buildmaster_host, port, slavename, passwd, basedir, 670 keepalive, usepty, umask=umask, maxdelay=maxdelay) 671 s.setServiceParent(application) 672 673 """ 674

675 -def createSlave(config):

676 m = Maker(config) 677 m.mkdir() 678 m.chdir() 679 if config['relocatable']: 680 config['basedir'] = '.' 681 try: 682 master = config['master'] 683 host, port = re.search(r'(.+):(\d+)', master).groups() 684 config['host'] = host 685 config['port'] = int(port) 686 except: 687 print "unparseable master location '%s'" % master 688 print " expecting something more like localhost:8007" 689 raise 690 contents = slaveTAC % config 691 692 m.makeTAC(contents, secret=True) 693 694 m.makefile() 695 m.mkinfo() 696 697 if not m.quiet: print "buildslave configured in %s" % m.basedir

698 699 700

701 -def stop(config, signame="TERM", wait=False):

702 import signal 703 basedir = config['basedir'] 704 quiet = config['quiet'] 705 os.chdir(basedir) 706 try: 707 f = open("twistd.pid", "rt") 708 except: 709 raise BuildbotNotRunningError 710 pid = int(f.read().strip()) 711 signum = getattr(signal, "SIG"+signame) 712 timer = 0 713 try: 714 os.kill(pid, signum) 715 except OSError, e: 716 if e.errno != 3: 717 raise 718 719 if not wait: 720 if not quiet: 721 print "sent SIG%s to process" % signame 722 return 723 time.sleep(0.1) 724 while timer < 10: 725 # poll once per second until twistd.pid goes away, up to 10 seconds 726 try: 727 os.kill(pid, 0) 728 except OSError: 729 if not quiet: 730 print "buildbot process %d is dead" % pid 731 return 732 timer += 1 733 time.sleep(1) 734 if not quiet: 735 print "never saw process go away"

736

737 -def restart(config):

738 quiet = config['quiet'] 739 from buildbot.scripts.startup import start 740 try: 741 stop(config, wait=True) 742 except BuildbotNotRunningError: 743 pass 744 if not quiet: 745 print "now restarting buildbot process.." 746 start(config)

747 748

749 -class StartOptions(MakerBase):

750 optFlags = [ 751 ['quiet', 'q', "Don't display startup log messages"], 752 ]

753 - def getSynopsis(self):

754 return "Usage: buildbot start [<basedir>]"

755

756 -class StopOptions(MakerBase):

757 - def getSynopsis(self):

758 return "Usage: buildbot stop [<basedir>]"

759

760 -class ReconfigOptions(MakerBase):

761 optFlags = [ 762 ['quiet', 'q', "Don't display log messages about reconfiguration"], 763 ]

764 - def getSynopsis(self):

765 return "Usage: buildbot reconfig [<basedir>]"

766 767 768

769 -class RestartOptions(MakerBase):

770 optFlags = [ 771 ['quiet', 'q', "Don't display startup log messages"], 772 ]

773 - def getSynopsis(self):

774 return "Usage: buildbot restart [<basedir>]"

775

776 -class DebugClientOptions(OptionsWithOptionsFile):

777 optFlags = [ 778 ['help', 'h', "Display this message"], 779 ] 780 optParameters = [ 781 ["master", "m", None, 782 "Location of the buildmaster's slaveport (host:port)"], 783 ["passwd", "p", None, "Debug password to use"], 784 ] 785 buildbotOptions = [ 786 [ 'debugMaster', 'passwd' ], 787 [ 'master', 'master' ], 788 ] 789

790 - def parseArgs(self, *args):

791 if len(args) > 0: 792 self['master'] = args[0] 793 if len(args) > 1: 794 self['passwd'] = args[1] 795 if len(args) > 2: 796 raise usage.UsageError("I wasn't expecting so many arguments")

797

798 -def debugclient(config):

799 from buildbot.clients import debug 800 801 master = config.get('master') 802 if master is None: 803 raise usage.UsageError("master must be specified: on the command " 804 "line or in ~/.buildbot/options") 805 806 passwd = config.get('passwd') 807 if passwd is None: 808 raise usage.UsageError("passwd must be specified: on the command " 809 "line or in ~/.buildbot/options") 810 811 d = debug.DebugWidget(master, passwd) 812 d.run()

813

814 -class StatusClientOptions(OptionsWithOptionsFile):

815 optFlags = [ 816 ['help', 'h', "Display this message"], 817 ] 818 optParameters = [ 819 ["master", "m", None, 820 "Location of the buildmaster's status port (host:port)"], 821 ["username", "u", "statusClient", "Username performing the trial build"], 822 ["passwd", None, "clientpw", "password for PB authentication"], 823 ] 824 buildbotOptions = [ 825 [ 'masterstatus', 'master' ], 826 ] 827

828 - def parseArgs(self, *args):

829 if len(args) > 0: 830 self['master'] = args[0] 831 if len(args) > 1: 832 raise usage.UsageError("I wasn't expecting so many arguments")

833

834 -def statuslog(config):

835 from buildbot.clients import base 836 master = config.get('master') 837 if master is None: 838 raise usage.UsageError("master must be specified: on the command " 839 "line or in ~/.buildbot/options") 840 passwd = config.get('passwd') 841 username = config.get('username') 842 c = base.TextClient(master, username=username, passwd=passwd) 843 c.run()

844

845 -def statusgui(config):

846 from buildbot.clients import gtkPanes 847 master = config.get('master') 848 if master is None: 849 raise usage.UsageError("master must be specified: on the command " 850 "line or in ~/.buildbot/options") 851 c = gtkPanes.GtkClient(master) 852 c.run()

853

854 -class SendChangeOptions(OptionsWithOptionsFile):

855 - def __init__(self):

856 OptionsWithOptionsFile.__init__(self) 857 self['properties'] = {}

858 859 optParameters = [ 860 ("master", "m", None, 861 "Location of the buildmaster's PBListener (host:port)"), 862 ("username", "u", None, "Username performing the commit"), 863 ("repository", "R", None, "Repository specifier"), 864 ("project", "P", None, "Project specifier"), 865 ("branch", "b", None, "Branch specifier"), 866 ("category", "c", None, "Category of repository"), 867 ("revision", "r", None, "Revision specifier"), 868 ("revision_file", None, None, "Filename containing revision spec"), 869 ("property", "p", None, 870 "A property for the change, in the format: name:value"), 871 ("comments", "m", None, "log message"), 872 ("logfile", "F", None, 873 "Read the log messages from this file (- for stdin)"), 874 ("when", "w", None, "timestamp to use as the change time"), 875 ] 876 877 buildbotOptions = [ 878 [ 'master', 'master' ], 879 [ 'username', 'username' ], 880 [ 'branch', 'branch' ], 881 [ 'category', 'category' ], 882 ] 883

884 - def getSynopsis(self):

885 return "Usage: buildbot sendchange [options] filenames.."

886 - def parseArgs(self, *args):

887 self['files'] = args

888 - def opt_property(self, property):

889 name,value = property.split(':') 890 self['properties'][name] = value

891 892

893 -def sendchange(config, runReactor=False):

894 """Send a single change to the buildmaster's PBChangeSource. The 895 connection will be drpoped as soon as the Change has been sent.""" 896 from buildbot.clients.sendchange import Sender 897 898 user = config.get('username') 899 master = config.get('master') 900 branch = config.get('branch') 901 category = config.get('category') 902 revision = config.get('revision') 903 properties = config.get('properties', {}) 904 repository = config.get('repository', '') 905 project = config.get('project', '') 906 if config.get('when'): 907 when = float(config.get('when')) 908 else: 909 when = None 910 if config.get("revision_file"): 911 revision = open(config["revision_file"],"r").read() 912 913 comments = config.get('comments') 914 if not comments and config.get('logfile'): 915 if config['logfile'] == "-": 916 f = sys.stdin 917 else: 918 f = open(config['logfile'], "rt") 919 comments = f.read() 920 if comments is None: 921 comments = "" 922 923 files = config.get('files', []) 924 925 assert user, "you must provide a username" 926 assert master, "you must provide the master location" 927 928 s = Sender(master, user) 929 d = s.send(branch, revision, comments, files, category=category, when=when, 930 properties=properties, repository=repository, project=project) 931 if runReactor: 932 d.addCallbacks(s.printSuccess, s.printFailure) 933 d.addBoth(s.stop) 934 s.run() 935 return d

936 937

938 -class ForceOptions(OptionsWithOptionsFile):

939 optParameters = [ 940 ["builder", None, None, "which Builder to start"], 941 ["branch", None, None, "which branch to build"], 942 ["revision", None, None, "which revision to build"], 943 ["reason", None, None, "the reason for starting the build"], 944 ] 945

946 - def parseArgs(self, *args):

947 args = list(args) 948 if len(args) > 0: 949 if self['builder'] is not None: 950 raise usage.UsageError("--builder provided in two ways") 951 self['builder'] = args.pop(0) 952 if len(args) > 0: 953 if self['reason'] is not None: 954 raise usage.UsageError("--reason provided in two ways") 955 self['reason'] = " ".join(args)

956 957

958 -class TryOptions(OptionsWithOptionsFile):

959 optParameters = [ 960 ["connect", "c", None, 961 "how to reach the buildmaster, either 'ssh' or 'pb'"], 962 # for ssh, use --tryhost, --username, and --trydir 963 ["tryhost", None, None, 964 "the hostname (used by ssh) for the buildmaster"], 965 ["trydir", None, None, 966 "the directory (on the tryhost) where tryjobs are deposited"], 967 ["username", "u", None, "Username performing the trial build"], 968 # for PB, use --master, --username, and --passwd 969 ["master", "m", None, 970 "Location of the buildmaster's PBListener (host:port)"], 971 ["passwd", None, None, "password for PB authentication"], 972 973 ["diff", None, None, 974 "Filename of a patch to use instead of scanning a local tree. Use '-' for stdin."], 975 ["patchlevel", "p", 0, 976 "Number of slashes to remove from patch pathnames, like the -p option to 'patch'"], 977 978 ["baserev", None, None, 979 "Base revision to use instead of scanning a local tree."], 980 981 ["vc", None, None, 982 "The VC system in use, one of: cvs,svn,tla,baz,darcs,p4"], 983 ["branch", None, None, 984 "The branch in use, for VC systems that can't figure it out" 985 " themselves"], 986 987 ["builder", "b", None, 988 "Run the trial build on this Builder. Can be used multiple times."], 989 ["properties", None, None, 990 "A set of properties made available in the build environment, format:prop=value,propb=valueb..."], 991 992 ["try-topfile", None, None, 993 "Name of a file at the top of the tree, used to find the top. Only needed for SVN and CVS."], 994 ["try-topdir", None, None, 995 "Path to the top of the working copy. Only needed for SVN and CVS."], 996 997 ] 998 999 optFlags = [ 1000 ["wait", None, "wait until the builds have finished"], 1001 ["dryrun", 'n', "Gather info, but don't actually submit."], 1002 ["get-builder-names", None, "Get the names of available builders. Doesn't submit anything. Only supported for 'pb' connections."], 1003 ] 1004 1005 # here it is, the definitive, quirky mapping of .buildbot/options names to 1006 # command-line options. Design by committee, anyone? 1007 buildbotOptions = [ 1008 [ 'try_connect', 'connect' ], 1009 #[ 'try_builders', 'builders' ], <-- handled in postOptions 1010 [ 'try_vc', 'vc' ], 1011 [ 'try_branch', 'branch' ], 1012 [ 'try_topdir', 'try-topdir' ], 1013 [ 'try_topfile', 'try-topfile' ], 1014 [ 'try_host', 'tryhost' ], 1015 [ 'try_username', 'username' ], 1016 [ 'try_dir', 'trydir' ], 1017 [ 'try_password', 'passwd' ], 1018 [ 'try_master', 'master' ], 1019 #[ 'try_wait', 'wait' ], <-- handled in postOptions 1020 [ 'masterstatus', 'master' ], 1021 ] 1022

1023 - def __init__(self):

1024 OptionsWithOptionsFile.__init__(self) 1025 self['builders'] = [] 1026 self['properties'] = {}

1027

1028 - def opt_builder(self, option):

1029 self['builders'].append(option)

1030

1031 - def opt_properties(self, option):

1032 # We need to split the value of this option into a dictionary of properties 1033 properties = {} 1034 propertylist = option.split(",") 1035 for i in range(0,len(propertylist)): 1036 print propertylist[i] 1037 splitproperty = propertylist[i].split("=") 1038 properties[splitproperty[0]] = splitproperty[1] 1039 self['properties'] = properties

1040

1041 - def opt_patchlevel(self, option):

1042 self['patchlevel'] = int(option)

1043

1044 - def getSynopsis(self):

1045 return "Usage: buildbot try [options]"

1046

1047 - def postOptions(self):

1048 opts = loadOptionsFile() 1049 if not self['builders']: 1050 self['builders'] = opts.get('try_builders', []) 1051 if opts.get('try_wait', False): 1052 self['wait'] = True

1053

1054 -def doTry(config):

1055 from buildbot.clients import tryclient 1056 t = tryclient.Try(config) 1057 t.run()

1058

1059 -class TryServerOptions(OptionsWithOptionsFile):

1060 optParameters = [ 1061 ["jobdir", None, None, "the jobdir (maildir) for submitting jobs"], 1062 ]

1063

1064 -def doTryServer(config):

1065 import md5 1066 jobdir = os.path.expanduser(config["jobdir"]) 1067 job = sys.stdin.read() 1068 # now do a 'safecat'-style write to jobdir/tmp, then move atomically to 1069 # jobdir/new . Rather than come up with a unique name randomly, I'm just 1070 # going to MD5 the contents and prepend a timestamp. 1071 timestring = "%d" % time.time() 1072 jobhash = md5.new(job).hexdigest() 1073 fn = "%s-%s" % (timestring, jobhash) 1074 tmpfile = os.path.join(jobdir, "tmp", fn) 1075 newfile = os.path.join(jobdir, "new", fn) 1076 f = open(tmpfile, "w") 1077 f.write(job) 1078 f.close() 1079 os.rename(tmpfile, newfile)

1080 1081

1082 -class CheckConfigOptions(OptionsWithOptionsFile):

1083 optFlags = [ 1084 ['quiet', 'q', "Don't display error messages or tracebacks"], 1085 ] 1086

1087 - def getSynopsis(self):

1088 return "Usage :buildbot checkconfig [configFile]\n" + \ 1089 " If not specified, 'master.cfg' will be used as 'configFile'"

1090

1091 - def parseArgs(self, *args):

1092 if len(args) >= 1: 1093 self['configFile'] = args[0] 1094 else: 1095 self['configFile'] = 'master.cfg'

1096 1097

1098 -def doCheckConfig(config):

1099 quiet = config.get('quiet') 1100 configFileName = config.get('configFile') 1101 try: 1102 from buildbot.scripts.checkconfig import ConfigLoader 1103 if os.path.isdir(configFileName): 1104 ConfigLoader(basedir=configFileName) 1105 else: 1106 ConfigLoader(configFileName=configFileName) 1107 except: 1108 if not quiet: 1109 # Print out the traceback in a nice format 1110 t, v, tb = sys.exc_info() 1111 traceback.print_exception(t, v, tb) 1112 sys.exit(1) 1113 1114 if not quiet: 1115 print "Config file is good!"

1116 1117

1118 -class Options(usage.Options):

1119 synopsis = "Usage: buildbot <command> [command options]" 1120 1121 subCommands = [ 1122 # the following are all admin commands 1123 ['create-master', None, MasterOptions, 1124 "Create and populate a directory for a new buildmaster"], 1125 ['upgrade-master', None, UpgradeMasterOptions, 1126 "Upgrade an existing buildmaster directory for the current version"], 1127 ['create-slave', None, SlaveOptions, 1128 "Create and populate a directory for a new buildslave"], 1129 ['start', None, StartOptions, "Start a buildmaster or buildslave"], 1130 ['stop', None, StopOptions, "Stop a buildmaster or buildslave"], 1131 ['restart', None, RestartOptions, 1132 "Restart a buildmaster or buildslave"], 1133 1134 ['reconfig', None, ReconfigOptions, 1135 "SIGHUP a buildmaster to make it re-read the config file"], 1136 ['sighup', None, ReconfigOptions, 1137 "SIGHUP a buildmaster to make it re-read the config file"], 1138 1139 ['sendchange', None, SendChangeOptions, 1140 "Send a change to the buildmaster"], 1141 1142 ['debugclient', None, DebugClientOptions, 1143 "Launch a small debug panel GUI"], 1144 1145 ['statuslog', None, StatusClientOptions, 1146 "Emit current builder status to stdout"], 1147 ['statusgui', None, StatusClientOptions, 1148 "Display a small window showing current builder status"], 1149 1150 #['force', None, ForceOptions, "Run a build"], 1151 ['try', None, TryOptions, "Run a build with your local changes"], 1152 1153 ['tryserver', None, TryServerOptions, 1154 "buildmaster-side 'try' support function, not for users"], 1155 1156 ['checkconfig', None, CheckConfigOptions, 1157 "test the validity of a master.cfg config file"], 1158 1159 # TODO: 'watch' 1160 ] 1161

1162 - def opt_version(self):

1163 import buildbot 1164 print "Buildbot version: %s" % buildbot.version 1165 usage.Options.opt_version(self)

1166

1167 - def opt_verbose(self):

1168 from twisted.python import log 1169 log.startLogging(sys.stderr)

1170

1171 - def postOptions(self):

1172 if not hasattr(self, 'subOptions'): 1173 raise usage.UsageError("must specify a command")

1174 1175

1176 -def run():

1177 config = Options() 1178 try: 1179 config.parseOptions() 1180 except usage.error, e: 1181 print "%s: %s" % (sys.argv[0], e) 1182 print 1183 c = getattr(config, 'subOptions', config) 1184 print str(c) 1185 sys.exit(1) 1186 1187 command = config.subCommand 1188 so = config.subOptions 1189 1190 if command == "create-master": 1191 createMaster(so) 1192 elif command == "upgrade-master": 1193 upgradeMaster(so) 1194 elif command == "create-slave": 1195 createSlave(so) 1196 elif command == "start": 1197 from buildbot.scripts.startup import start 1198 start(so) 1199 elif command == "stop": 1200 stop(so, wait=True) 1201 elif command == "restart": 1202 restart(so) 1203 elif command == "reconfig" or command == "sighup": 1204 from buildbot.scripts.reconfig import Reconfigurator 1205 Reconfigurator().run(so) 1206 elif command == "sendchange": 1207 sendchange(so, True) 1208 elif command == "debugclient": 1209 debugclient(so) 1210 elif command == "statuslog": 1211 statuslog(so) 1212 elif command == "statusgui": 1213 statusgui(so) 1214 elif command == "try": 1215 doTry(so) 1216 elif command == "tryserver": 1217 doTryServer(so) 1218 elif command == "checkconfig": 1219 doCheckConfig(so) 1220 sys.exit(0)

1221

Source Code for Module buildbot.scripts.runner