Package buildbot :: Package scripts :: Module runner
[frames] | no frames]

Source Code for Module buildbot.scripts.runner

   1  # This file is part of Buildbot.  Buildbot is free software: you can 
   2  # redistribute it and/or modify it under the terms of the GNU General Public 
   3  # License as published by the Free Software Foundation, version 2. 
   4  # 
   5  # This program is distributed in the hope that it will be useful, but WITHOUT 
   6  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
   7  # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more 
   8  # details. 
   9  # 
  10  # You should have received a copy of the GNU General Public License along with 
  11  # this program; if not, write to the Free Software Foundation, Inc., 51 
  12  # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 
  13  # 
  14  # Copyright Buildbot Team Members 
  15   
  16   
  17  # N.B.: don't import anything that might pull in a reactor yet. Some of our 
  18  # subcommands want to load modules that need the gtk reactor. 
  19  # 
  20  # Also don't forget to mirror your changes on command-line options in manual 
  21  # pages and texinfo documentation. 
  22   
  23  import copy 
  24  import os, sys, stat, re, time 
  25  from twisted.python import usage, util, runtime 
  26  from twisted.internet import defer 
  27   
  28  from buildbot.interfaces import BuildbotNotRunningError 
29 30 -def in_reactor(f):
31 """decorate a function by running it with maybeDeferred in a reactor""" 32 def wrap(*args, **kwargs): 33 from twisted.internet import reactor 34 result = [ ] 35 def async(): 36 d = defer.maybeDeferred(f, *args, **kwargs) 37 def eb(f): 38 f.printTraceback()
39 d.addErrback(eb) 40 def do_stop(r): 41 result.append(r) 42 reactor.stop() 43 d.addBoth(do_stop) 44 reactor.callWhenRunning(async) 45 reactor.run() 46 return result[0] 47 wrap.__doc__ = f.__doc__ 48 wrap.__name__ = f.__name__ 49 wrap._orig = f # for tests 50 return wrap 51
52 -def isBuildmasterDir(dir):
53 buildbot_tac = os.path.join(dir, "buildbot.tac") 54 if not os.path.isfile(buildbot_tac): 55 print "no buildbot.tac" 56 return False 57 58 contents = open(buildbot_tac, "r").read() 59 return "Application('buildmaster')" in contents
60
61 # the create/start/stop commands should all be run as the same user, 62 # preferably a separate 'buildbot' account. 63 64 # Note that the terms 'options' and 'config' are used intechangeably here - in 65 # fact, they are intercanged several times. Caveat legator. 66 67 -class OptionsWithOptionsFile(usage.Options):
68 # subclasses should set this to a list-of-lists in order to source the 69 # .buildbot/options file. 70 # buildbotOptions = [ [ 'optfile-name', 'option-name' ], .. ] 71 buildbotOptions = None 72
73 - def __init__(self, *args):
74 # for options in self.buildbotOptions, optParameters, and the options 75 # file, change the default in optParameters *before* calling through 76 # to the parent constructor 77 78 # Options uses reflect.accumulateClassList, so this *must* be 79 # a class attribute; however, we do not want to permanently change 80 # the class. So we patch it temporarily and restore it after. 81 cls = self.__class__ 82 if hasattr(cls, 'optParameters'): 83 old_optParameters = cls.optParameters 84 cls.optParameters = op = copy.deepcopy(cls.optParameters) 85 if self.buildbotOptions: 86 optfile = loadOptionsFile() 87 for optfile_name, option_name in self.buildbotOptions: 88 for i in range(len(op)): 89 if (op[i][0] == option_name and optfile_name in optfile): 90 op[i] = list(op[i]) 91 op[i][2] = optfile[optfile_name] 92 usage.Options.__init__(self, *args) 93 if hasattr(cls, 'optParameters'): 94 cls.optParameters = old_optParameters
95
96 -def loadOptionsFile():
97 """Find the .buildbot/FILENAME file. Crawl from the current directory up 98 towards the root, and also look in ~/.buildbot . The first directory 99 that's owned by the user and has the file we're looking for wins. Windows 100 skips the owned-by-user test. 101 102 @rtype: dict 103 @return: a dictionary of names defined in the options file. If no options 104 file was found, return an empty dict. 105 """ 106 107 here = os.path.abspath(os.getcwd()) 108 109 if runtime.platformType == 'win32': 110 # never trust env-vars, use the proper API 111 from win32com.shell import shellcon, shell 112 appdata = shell.SHGetFolderPath(0, shellcon.CSIDL_APPDATA, 0, 0) 113 home = os.path.join(appdata, "buildbot") 114 else: 115 home = os.path.expanduser("~/.buildbot") 116 117 searchpath = [] 118 toomany = 20 119 while True: 120 searchpath.append(os.path.join(here, ".buildbot")) 121 next = os.path.dirname(here) 122 if next == here: 123 break # we've hit the root 124 here = next 125 toomany -= 1 # just in case 126 if toomany == 0: 127 raise ValueError("Hey, I seem to have wandered up into the " 128 "infinite glories of the heavens. Oops.") 129 searchpath.append(home) 130 131 localDict = {} 132 133 for d in searchpath: 134 if os.path.isdir(d): 135 if runtime.platformType != 'win32': 136 if os.stat(d)[stat.ST_UID] != os.getuid(): 137 print "skipping %s because you don't own it" % d 138 continue # security, skip other people's directories 139 optfile = os.path.join(d, "options") 140 if os.path.exists(optfile): 141 try: 142 f = open(optfile, "r") 143 options = f.read() 144 exec options in localDict 145 except: 146 print "error while reading %s" % optfile 147 raise 148 break 149 150 for k in localDict.keys(): 151 if k.startswith("__"): 152 del localDict[k] 153 return localDict
154
155 -class MakerBase(OptionsWithOptionsFile):
156 optFlags = [ 157 ['help', 'h', "Display this message"], 158 ["quiet", "q", "Do not emit the commands being run"], 159 ] 160 161 usage.Options.longdesc = """ 162 Operates upon the specified <basedir> (or the current directory, if not 163 specified). 164 """ 165 166 opt_h = usage.Options.opt_help 167
168 - def parseArgs(self, *args):
169 if len(args) > 0: 170 self['basedir'] = args[0] 171 else: 172 # Use the current directory if no basedir was specified. 173 self['basedir'] = os.getcwd() 174 if len(args) > 1: 175 raise usage.UsageError("I wasn't expecting so many arguments")
176
177 - def postOptions(self):
178 self['basedir'] = os.path.abspath(self['basedir'])
179
180 -class Maker:
181 - def __init__(self, config):
182 self.config = config 183 self.basedir = config['basedir'] 184 self.force = config.get('force', False) 185 self.quiet = config['quiet']
186
187 - def mkdir(self):
188 if os.path.exists(self.basedir): 189 if not self.quiet: 190 print "updating existing installation" 191 return 192 if not self.quiet: print "mkdir", self.basedir 193 os.mkdir(self.basedir)
194
195 - def chdir(self):
196 os.chdir(self.basedir)
197
198 - def makeTAC(self, contents, secret=False):
199 tacfile = "buildbot.tac" 200 if os.path.exists(tacfile): 201 oldcontents = open(tacfile, "rt").read() 202 if oldcontents == contents: 203 if not self.quiet: 204 print "buildbot.tac already exists and is correct" 205 return 206 if not self.quiet: 207 print "not touching existing buildbot.tac" 208 print "creating buildbot.tac.new instead" 209 tacfile = "buildbot.tac.new" 210 f = open(tacfile, "wt") 211 f.write(contents) 212 f.close() 213 if secret: 214 os.chmod(tacfile, 0600)
215
216 - def sampleconfig(self, source):
217 target = "master.cfg.sample" 218 if not self.quiet: 219 print "creating %s" % target 220 config_sample = open(source, "rt").read() 221 if self.config['db']: 222 config_sample = config_sample.replace('sqlite:///state.sqlite', 223 self.config['db']) 224 f = open(target, "wt") 225 f.write(config_sample) 226 f.close() 227 os.chmod(target, 0600)
228
229 - def public_html(self, files):
230 webdir = os.path.join(self.basedir, "public_html") 231 if os.path.exists(webdir): 232 if not self.quiet: 233 print "public_html/ already exists: not replacing" 234 return 235 else: 236 os.mkdir(webdir) 237 if not self.quiet: 238 print "populating public_html/" 239 for target, source in files.iteritems(): 240 target = os.path.join(webdir, target) 241 f = open(target, "wt") 242 f.write(open(source, "rt").read()) 243 f.close()
244
245 - def create_db(self):
246 from buildbot.db import connector 247 from buildbot.master import BuildMaster 248 from buildbot import config as config_module 249 250 from buildbot import monkeypatches 251 monkeypatches.patch_all() 252 253 # create a master with the default configuration, but with db_url 254 # overridden 255 master_cfg = config_module.MasterConfig() 256 master_cfg.db['db_url'] = self.config['db'] 257 master = BuildMaster(self.basedir) 258 master.config = master_cfg 259 db = connector.DBConnector(master, self.basedir) 260 d = db.setup(check_version=False, verbose=not self.config['quiet']) 261 if not self.config['quiet']: 262 print "creating database (%s)" % (master_cfg.db['db_url'],) 263 d = db.model.upgrade() 264 return d
265
266 - def populate_if_missing(self, target, source, overwrite=False):
267 new_contents = open(source, "rt").read() 268 if os.path.exists(target): 269 old_contents = open(target, "rt").read() 270 if old_contents != new_contents: 271 if overwrite: 272 if not self.quiet: 273 print "%s has old/modified contents" % target 274 print " overwriting it with new contents" 275 open(target, "wt").write(new_contents) 276 else: 277 if not self.quiet: 278 print "%s has old/modified contents" % target 279 print " writing new contents to %s.new" % target 280 open(target + ".new", "wt").write(new_contents) 281 # otherwise, it's up to date 282 else: 283 if not self.quiet: 284 print "populating %s" % target 285 open(target, "wt").write(new_contents)
286
287 - def move_if_present(self, source, dest):
288 if os.path.exists(source): 289 if os.path.exists(dest): 290 print "Notice: %s now overrides %s" % (dest, source) 291 print " as the latter is not used by buildbot anymore." 292 print " Decide which one you want to keep." 293 else: 294 try: 295 print "Notice: Moving %s to %s." % (source, dest) 296 print " You can (and probably want to) remove it if you haven't modified this file." 297 os.renames(source, dest) 298 except Exception, e: 299 print "Error moving %s to %s: %s" % (source, dest, str(e))
300
301 - def upgrade_public_html(self, files):
302 webdir = os.path.join(self.basedir, "public_html") 303 if not os.path.exists(webdir): 304 if not self.quiet: 305 print "populating public_html/" 306 os.mkdir(webdir) 307 for target, source in files.iteritems(): 308 self.populate_if_missing(os.path.join(webdir, target), 309 source)
310
311 - def check_master_cfg(self, expected_db_url=None):
312 """Check the buildmaster configuration, returning an exit status (so 313 0=success)."""
314 315 DB_HELP = """ 316 The --db string is evaluated to build the DB object, which specifies 317 which database the buildmaster should use to hold scheduler state and 318 status information. The default (which creates an SQLite database in 319 BASEDIR/state.sqlite) is equivalent to: 320 321 --db='sqlite:///state.sqlite' 322 323 To use a remote MySQL database instead, use something like: 324 325 --db='mysql://bbuser:bbpasswd@dbhost/bbdb' 326 """
327 328 -class UpgradeMasterOptions(MakerBase):
329 optFlags = [ 330 ["replace", "r", "Replace any modified files without confirmation."], 331 ] 332 optParameters = [ 333 ] 334
335 - def getSynopsis(self):
336 return "Usage: buildbot upgrade-master [options] [<basedir>]"
337 338 longdesc = """ 339 This command takes an existing buildmaster working directory and 340 adds/modifies the files there to work with the current version of 341 buildbot. When this command is finished, the buildmaster directory should 342 look much like a brand-new one created by the 'create-master' command. 343 344 Use this after you've upgraded your buildbot installation and before you 345 restart the buildmaster to use the new version. 346 347 If you have modified the files in your working directory, this command 348 will leave them untouched, but will put the new recommended contents in a 349 .new file (for example, if index.html has been modified, this command 350 will create index.html.new). You can then look at the new version and 351 decide how to merge its contents into your modified file. 352 353 When upgrading from a pre-0.8.0 release (which did not use a database), 354 this command will create the given database and migrate data from the old 355 pickle files into it, then move the pickle files out of the way (e.g. to 356 changes.pck.old). 357 358 When upgrading the database, this command uses the database specified in 359 the master configuration file. If you wish to use a database other than 360 the default (sqlite), be sure to set that parameter before upgrading. 361 """
362
363 @in_reactor 364 @defer.deferredGenerator 365 -def upgradeMaster(config):
366 from buildbot import config as config_module 367 from buildbot import monkeypatches 368 import traceback 369 370 monkeypatches.patch_all() 371 372 m = Maker(config) 373 basedir = os.path.expanduser(config['basedir']) 374 375 if runtime.platformType != 'win32': # no pids on win32 376 if not config['quiet']: print "checking for running master" 377 pidfile = os.path.join(basedir, 'twistd.pid') 378 if os.path.exists(pidfile): 379 print "'%s' exists - is this master still running?" % (pidfile,) 380 yield 1 381 return 382 383 if not config['quiet']: print "checking master.cfg" 384 try: 385 master_cfg = config_module.MasterConfig.loadConfig( 386 basedir, 'master.cfg') 387 except config_module.ConfigErrors, e: 388 print "Errors loading configuration:" 389 for msg in e.errors: 390 print " " + msg 391 yield 1 392 return 393 except: 394 print "Errors loading configuration:" 395 traceback.print_exc() 396 yield 1 397 return 398 399 if not config['quiet']: print "upgrading basedir" 400 basedir = os.path.expanduser(config['basedir']) 401 # TODO: check TAC file 402 # check web files: index.html, default.css, robots.txt 403 m.chdir() 404 m.upgrade_public_html({ 405 'bg_gradient.jpg' : util.sibpath(__file__, "../status/web/files/bg_gradient.jpg"), 406 'default.css' : util.sibpath(__file__, "../status/web/files/default.css"), 407 'robots.txt' : util.sibpath(__file__, "../status/web/files/robots.txt"), 408 'favicon.ico' : util.sibpath(__file__, "../status/web/files/favicon.ico"), 409 }) 410 m.populate_if_missing(os.path.join(basedir, "master.cfg.sample"), 411 util.sibpath(__file__, "sample.cfg"), 412 overwrite=True) 413 # if index.html exists, use it to override the root page tempalte 414 m.move_if_present(os.path.join(basedir, "public_html/index.html"), 415 os.path.join(basedir, "templates/root.html")) 416 417 from buildbot.db import connector 418 from buildbot.master import BuildMaster 419 420 if not config['quiet']: 421 print "upgrading database (%s)" % (master_cfg.db['db_url']) 422 master = BuildMaster(config['basedir']) 423 master.config = master_cfg 424 db = connector.DBConnector(master, basedir=config['basedir']) 425 426 wfd = defer.waitForDeferred( 427 db.setup(check_version=False, verbose=not config['quiet'])) 428 yield wfd 429 wfd.getResult() 430 431 wfd = defer.waitForDeferred( 432 db.model.upgrade()) 433 yield wfd 434 wfd.getResult() 435 436 if not config['quiet']: print "upgrade complete" 437 yield 0
438
439 440 -class MasterOptions(MakerBase):
441 optFlags = [ 442 ["force", "f", 443 "Re-use an existing directory (will not overwrite master.cfg file)"], 444 ["relocatable", "r", 445 "Create a relocatable buildbot.tac"], 446 ["no-logrotate", "n", 447 "Do not permit buildmaster rotate logs by itself"] 448 ] 449 optParameters = [ 450 ["config", "c", "master.cfg", "name of the buildmaster config file"], 451 ["log-size", "s", "10000000", 452 "size at which to rotate twisted log files"], 453 ["log-count", "l", "10", 454 "limit the number of kept old twisted log files"], 455 ["db", None, "sqlite:///state.sqlite", 456 "which DB to use for scheduler/status state. See below for syntax."], 457 ]
458 - def getSynopsis(self):
459 return "Usage: buildbot create-master [options] [<basedir>]"
460 461 longdesc = """ 462 This command creates a buildmaster working directory and buildbot.tac file. 463 The master will live in <dir> and create various files there. If 464 --relocatable is given, then the resulting buildbot.tac file will be 465 written such that its containing directory is assumed to be the basedir. 466 This is generally a good idea. 467 468 At runtime, the master will read a configuration file (named 469 'master.cfg' by default) in its basedir. This file should contain python 470 code which eventually defines a dictionary named 'BuildmasterConfig'. 471 The elements of this dictionary are used to configure the Buildmaster. 472 See doc/config.xhtml for details about what can be controlled through 473 this interface. 474 """ + DB_HELP + """ 475 The --db string is stored verbatim in the buildbot.tac file, and 476 evaluated as 'buildbot start' time to pass a DBConnector instance into 477 the newly-created BuildMaster object. 478 """ 479
480 - def postOptions(self):
481 MakerBase.postOptions(self) 482 if not re.match('^\d+$', self['log-size']): 483 raise usage.UsageError("log-size parameter needs to be an int") 484 if not re.match('^\d+$', self['log-count']) and \ 485 self['log-count'] != 'None': 486 raise usage.UsageError("log-count parameter needs to be an int "+ 487 " or None")
488 489 490 masterTACTemplate = [""" 491 import os 492 493 from twisted.application import service 494 from buildbot.master import BuildMaster 495 496 basedir = r'%(basedir)s' 497 rotateLength = %(log-size)s 498 maxRotatedFiles = %(log-count)s 499 500 # if this is a relocatable tac file, get the directory containing the TAC 501 if basedir == '.': 502 import os.path 503 basedir = os.path.abspath(os.path.dirname(__file__)) 504 505 # note: this line is matched against to check that this is a buildmaster 506 # directory; do not edit it. 507 application = service.Application('buildmaster') 508 """, 509 """ 510 try: 511 from twisted.python.logfile import LogFile 512 from twisted.python.log import ILogObserver, FileLogObserver 513 logfile = LogFile.fromFullPath(os.path.join(basedir, "twistd.log"), rotateLength=rotateLength, 514 maxRotatedFiles=maxRotatedFiles) 515 application.setComponent(ILogObserver, FileLogObserver(logfile).emit) 516 except ImportError: 517 # probably not yet twisted 8.2.0 and beyond, can't set log yet 518 pass 519 """, 520 """ 521 configfile = r'%(config)s' 522 523 m = BuildMaster(basedir, configfile) 524 m.setServiceParent(application) 525 m.log_rotation.rotateLength = rotateLength 526 m.log_rotation.maxRotatedFiles = maxRotatedFiles 527 528 """]
529 530 @in_reactor 531 -def createMaster(config):
532 m = Maker(config) 533 m.mkdir() 534 m.chdir() 535 if config['relocatable']: 536 config['basedir'] = '.' 537 if config['no-logrotate']: 538 masterTAC = "".join([masterTACTemplate[0]] + masterTACTemplate[2:]) 539 else: 540 masterTAC = "".join(masterTACTemplate) 541 contents = masterTAC % config 542 m.makeTAC(contents) 543 m.sampleconfig(util.sibpath(__file__, "sample.cfg")) 544 m.public_html({ 545 'bg_gradient.jpg' : util.sibpath(__file__, "../status/web/files/bg_gradient.jpg"), 546 'default.css' : util.sibpath(__file__, "../status/web/files/default.css"), 547 'robots.txt' : util.sibpath(__file__, "../status/web/files/robots.txt"), 548 'favicon.ico' : util.sibpath(__file__, "../status/web/files/favicon.ico"), 549 }) 550 d = m.create_db() 551 552 def print_status(r): 553 if not m.quiet: 554 print "buildmaster configured in %s" % m.basedir
555 d.addCallback(print_status) 556 return d 557
558 -def stop(config, signame="TERM", wait=False):
559 import signal 560 basedir = config['basedir'] 561 quiet = config['quiet'] 562 563 if not isBuildmasterDir(config['basedir']): 564 print "not a buildmaster directory" 565 sys.exit(1) 566 567 os.chdir(basedir) 568 try: 569 f = open("twistd.pid", "rt") 570 except: 571 raise BuildbotNotRunningError 572 pid = int(f.read().strip()) 573 signum = getattr(signal, "SIG"+signame) 574 timer = 0 575 try: 576 os.kill(pid, signum) 577 except OSError, e: 578 if e.errno != 3: 579 raise 580 581 if not wait: 582 if not quiet: 583 print "sent SIG%s to process" % signame 584 return 585 time.sleep(0.1) 586 while timer < 10: 587 # poll once per second until twistd.pid goes away, up to 10 seconds 588 try: 589 os.kill(pid, 0) 590 except OSError: 591 if not quiet: 592 print "buildbot process %d is dead" % pid 593 return 594 timer += 1 595 time.sleep(1) 596 if not quiet: 597 print "never saw process go away"
598
599 -def restart(config):
600 basedir = config['basedir'] 601 quiet = config['quiet'] 602 603 if not isBuildmasterDir(basedir): 604 print "not a buildmaster directory" 605 sys.exit(1) 606 607 from buildbot.scripts.startup import start 608 try: 609 stop(config, wait=True) 610 except BuildbotNotRunningError: 611 pass 612 if not quiet: 613 print "now restarting buildbot process.." 614 start(config)
615
616 617 -class StartOptions(MakerBase):
618 optFlags = [ 619 ['quiet', 'q', "Don't display startup log messages"], 620 ]
621 - def getSynopsis(self):
622 return "Usage: buildbot start [<basedir>]"
623
624 -class StopOptions(MakerBase):
625 - def getSynopsis(self):
626 return "Usage: buildbot stop [<basedir>]"
627
628 -class ReconfigOptions(MakerBase):
629 optFlags = [ 630 ['quiet', 'q', "Don't display log messages about reconfiguration"], 631 ]
632 - def getSynopsis(self):
633 return "Usage: buildbot reconfig [<basedir>]"
634
635 636 637 -class RestartOptions(MakerBase):
638 optFlags = [ 639 ['quiet', 'q', "Don't display startup log messages"], 640 ]
641 - def getSynopsis(self):
642 return "Usage: buildbot restart [<basedir>]"
643
644 -class DebugClientOptions(OptionsWithOptionsFile):
645 optFlags = [ 646 ['help', 'h', "Display this message"], 647 ] 648 optParameters = [ 649 ["master", "m", None, 650 "Location of the buildmaster's slaveport (host:port)"], 651 ["passwd", "p", None, "Debug password to use"], 652 ] 653 buildbotOptions = [ 654 [ 'debugMaster', 'passwd' ], 655 [ 'master', 'master' ], 656 ]
657 - def getSynopsis(self):
658 return "Usage: buildbot debugclient [options]"
659
660 - def parseArgs(self, *args):
661 if len(args) > 0: 662 self['master'] = args[0] 663 if len(args) > 1: 664 self['passwd'] = args[1] 665 if len(args) > 2: 666 raise usage.UsageError("I wasn't expecting so many arguments")
667
668 -def debugclient(config):
669 from buildbot.clients import debug 670 671 master = config.get('master') 672 if master is None: 673 raise usage.UsageError("master must be specified: on the command " 674 "line or in ~/.buildbot/options") 675 676 passwd = config.get('passwd') 677 if passwd is None: 678 raise usage.UsageError("passwd must be specified: on the command " 679 "line or in ~/.buildbot/options") 680 681 d = debug.DebugWidget(master, passwd) 682 d.run()
683
684 -class StatusClientOptions(OptionsWithOptionsFile):
685 optFlags = [ 686 ['help', 'h', "Display this message"], 687 ] 688 optParameters = [ 689 ["master", "m", None, 690 "Location of the buildmaster's status port (host:port)"], 691 ["username", "u", "statusClient", "Username performing the trial build"], 692 ["passwd", None, "clientpw", "password for PB authentication"], 693 ] 694 buildbotOptions = [ 695 [ 'masterstatus', 'master' ], 696 ] 697
698 - def parseArgs(self, *args):
699 if len(args) > 0: 700 self['master'] = args[0] 701 if len(args) > 1: 702 raise usage.UsageError("I wasn't expecting so many arguments")
703
704 -class StatusLogOptions(StatusClientOptions):
705 - def getSynopsis(self):
706 return "Usage: buildbot statuslog [options]"
707
708 -class StatusGuiOptions(StatusClientOptions):
709 - def getSynopsis(self):
710 return "Usage: buildbot statusgui [options]"
711
712 -def statuslog(config):
713 from buildbot.clients import base 714 master = config.get('master') 715 if master is None: 716 raise usage.UsageError("master must be specified: on the command " 717 "line or in ~/.buildbot/options") 718 passwd = config.get('passwd') 719 username = config.get('username') 720 c = base.TextClient(master, username=username, passwd=passwd) 721 c.run()
722
723 -def statusgui(config):
724 from buildbot.clients import gtkPanes 725 master = config.get('master') 726 if master is None: 727 raise usage.UsageError("master must be specified: on the command " 728 "line or in ~/.buildbot/options") 729 passwd = config.get('passwd') 730 username = config.get('username') 731 c = gtkPanes.GtkClient(master, username=username, passwd=passwd) 732 c.run()
733
734 -class SendChangeOptions(OptionsWithOptionsFile):
735 - def __init__(self):
736 OptionsWithOptionsFile.__init__(self) 737 self['properties'] = {}
738 739 optParameters = [ 740 ("master", "m", None, 741 "Location of the buildmaster's PBListener (host:port)"), 742 # deprecated in 0.8.3; remove in 0.8.5 (bug #1711) 743 ("auth", "a", None, "Authentication token - username:password, or prompt for password"), 744 ("who", "W", None, "Author of the commit"), 745 ("repository", "R", '', "Repository specifier"), 746 ("vc", "s", None, "The VC system in use, one of: cvs, svn, darcs, hg, " 747 "bzr, git, mtn, p4"), 748 ("project", "P", '', "Project specifier"), 749 ("branch", "b", None, "Branch specifier"), 750 ("category", "C", None, "Category of repository"), 751 ("revision", "r", None, "Revision specifier"), 752 ("revision_file", None, None, "Filename containing revision spec"), 753 ("property", "p", None, 754 "A property for the change, in the format: name:value"), 755 ("comments", "c", None, "log message"), 756 ("logfile", "F", None, 757 "Read the log messages from this file (- for stdin)"), 758 ("when", "w", None, "timestamp to use as the change time"), 759 ("revlink", "l", '', "Revision link (revlink)"), 760 ("encoding", "e", 'utf8', 761 "Encoding of other parameters (default utf8)"), 762 ] 763 764 buildbotOptions = [ 765 [ 'master', 'master' ], 766 [ 'who', 'who' ], 767 [ 'branch', 'branch' ], 768 [ 'category', 'category' ], 769 [ 'vc', 'vc' ], 770 ] 771
772 - def getSynopsis(self):
773 return "Usage: buildbot sendchange [options] filenames.."
774 - def parseArgs(self, *args):
775 self['files'] = args
776 - def opt_property(self, property):
777 name,value = property.split(':', 1) 778 self['properties'][name] = value
779
780 781 -def sendchange(config, runReactor=False):
782 """Send a single change to the buildmaster's PBChangeSource. The 783 connection will be drpoped as soon as the Change has been sent.""" 784 from buildbot.clients import sendchange 785 786 encoding = config.get('encoding', 'utf8') 787 who = config.get('who') 788 auth = config.get('auth') 789 master = config.get('master') 790 branch = config.get('branch') 791 category = config.get('category') 792 revision = config.get('revision') 793 properties = config.get('properties', {}) 794 repository = config.get('repository', '') 795 vc = config.get('vc', None) 796 project = config.get('project', '') 797 revlink = config.get('revlink', '') 798 if config.get('when'): 799 when = float(config.get('when')) 800 else: 801 when = None 802 if config.get("revision_file"): 803 revision = open(config["revision_file"],"r").read() 804 805 comments = config.get('comments') 806 if not comments and config.get('logfile'): 807 if config['logfile'] == "-": 808 f = sys.stdin 809 else: 810 f = open(config['logfile'], "rt") 811 comments = f.read() 812 if comments is None: 813 comments = "" 814 815 files = config.get('files', ()) 816 817 vcs = ['cvs', 'svn', 'darcs', 'hg', 'bzr', 'git', 'mtn', 'p4', None] 818 assert vc in vcs, "vc must be 'cvs', 'svn', 'darcs', 'hg', 'bzr', " \ 819 "'git', 'mtn', or 'p4'" 820 821 # fix up the auth with a password if none was given 822 if not auth: 823 auth = 'change:changepw' 824 if ':' not in auth: 825 import getpass 826 pw = getpass.getpass("Enter password for '%s': " % auth) 827 auth = "%s:%s" % (auth, pw) 828 auth = auth.split(':', 1) 829 830 assert who, "you must provide a committer (--who)" 831 assert master, "you must provide the master location" 832 833 s = sendchange.Sender(master, auth, encoding=encoding) 834 d = s.send(branch, revision, comments, files, who=who, category=category, when=when, 835 properties=properties, repository=repository, vc=vc, project=project, 836 revlink=revlink) 837 838 if runReactor: 839 from twisted.internet import reactor 840 status = [True] 841 def printSuccess(_): 842 print "change sent successfully"
843 def failed(f): 844 status[0] = False 845 print "change NOT sent - something went wrong: " + str(f) 846 d.addCallbacks(printSuccess, failed) 847 d.addBoth(lambda _ : reactor.stop()) 848 reactor.run() 849 return status[0] 850 return d 851
852 853 -class ForceOptions(OptionsWithOptionsFile):
854 optParameters = [ 855 ["builder", None, None, "which Builder to start"], 856 ["branch", None, None, "which branch to build"], 857 ["revision", None, None, "which revision to build"], 858 ["reason", None, None, "the reason for starting the build"], 859 ["props", None, None, 860 "A set of properties made available in the build environment, " 861 "format is --properties=prop1=value1,prop2=value2,.. " 862 "option can be specified multiple times."], 863 ] 864
865 - def parseArgs(self, *args):
866 args = list(args) 867 if len(args) > 0: 868 if self['builder'] is not None: 869 raise usage.UsageError("--builder provided in two ways") 870 self['builder'] = args.pop(0) 871 if len(args) > 0: 872 if self['reason'] is not None: 873 raise usage.UsageError("--reason provided in two ways") 874 self['reason'] = " ".join(args)
875
876 877 -class TryOptions(OptionsWithOptionsFile):
878 optParameters = [ 879 ["connect", "c", None, 880 "How to reach the buildmaster, either 'ssh' or 'pb'"], 881 # for ssh, use --host, --username, and --jobdir 882 ["host", None, None, 883 "Hostname (used by ssh) for the buildmaster"], 884 ["jobdir", None, None, 885 "Directory (on the buildmaster host) where try jobs are deposited"], 886 ["username", "u", None, 887 "Username performing the try build"], 888 # for PB, use --master, --username, and --passwd 889 ["master", "m", None, 890 "Location of the buildmaster's PBListener (host:port)"], 891 ["passwd", None, None, 892 "Password for PB authentication"], 893 ["who", "w", None, 894 "Who is responsible for the try build"], 895 ["comment", "C", None, 896 "A comment which can be used in notifications for this build"], 897 898 ["diff", None, None, 899 "Filename of a patch to use instead of scanning a local tree. " 900 "Use '-' for stdin."], 901 ["patchlevel", "p", 0, 902 "Number of slashes to remove from patch pathnames, " 903 "like the -p option to 'patch'"], 904 905 ["baserev", None, None, 906 "Base revision to use instead of scanning a local tree."], 907 908 ["vc", None, None, 909 "The VC system in use, one of: bzr, cvs, darcs, git, hg, " 910 "mtn, p4, svn"], 911 ["branch", None, None, 912 "The branch in use, for VC systems that can't figure it out " 913 "themselves"], 914 ["repository", None, None, 915 "Repository to use, instead of path to working directory."], 916 917 ["builder", "b", None, 918 "Run the trial build on this Builder. Can be used multiple times."], 919 ["properties", None, None, 920 "A set of properties made available in the build environment, " 921 "format is --properties=prop1=value1,prop2=value2,.. " 922 "option can be specified multiple times."], 923 924 ["topfile", None, None, 925 "Name of a file at the top of the tree, used to find the top. " 926 "Only needed for SVN and CVS."], 927 ["topdir", None, None, 928 "Path to the top of the working copy. Only needed for SVN and CVS."], 929 ] 930 931 optFlags = [ 932 ["wait", None, 933 "wait until the builds have finished"], 934 ["dryrun", 'n', 935 "Gather info, but don't actually submit."], 936 ["get-builder-names", None, 937 "Get the names of available builders. Doesn't submit anything. " 938 "Only supported for 'pb' connections."], 939 ["quiet", "q", 940 "Don't print status of current builds while waiting."], 941 ] 942 943 # Mapping of .buildbot/options names to command-line options 944 buildbotOptions = [ 945 [ 'try_connect', 'connect' ], 946 #[ 'try_builders', 'builders' ], <-- handled in postOptions 947 [ 'try_vc', 'vc' ], 948 [ 'try_branch', 'branch' ], 949 [ 'try_repository', 'repository' ], 950 [ 'try_topdir', 'topdir' ], 951 [ 'try_topfile', 'topfile' ], 952 [ 'try_host', 'host' ], 953 [ 'try_username', 'username' ], 954 [ 'try_jobdir', 'jobdir' ], 955 [ 'try_passwd', 'passwd' ], 956 [ 'try_master', 'master' ], 957 [ 'try_who', 'who' ], 958 [ 'try_comment', 'comment' ], 959 #[ 'try_wait', 'wait' ], <-- handled in postOptions 960 #[ 'try_quiet', 'quiet' ], <-- handled in postOptions 961 962 # Deprecated command mappings from the quirky old days: 963 [ 'try_masterstatus', 'master' ], 964 [ 'try_dir', 'jobdir' ], 965 [ 'try_password', 'passwd' ], 966 ] 967
968 - def __init__(self):
969 OptionsWithOptionsFile.__init__(self) 970 self['builders'] = [] 971 self['properties'] = {}
972
973 - def opt_builder(self, option):
974 self['builders'].append(option)
975
976 - def opt_properties(self, option):
977 # We need to split the value of this option into a dictionary of properties 978 propertylist = option.split(",") 979 for i in range(0,len(propertylist)): 980 splitproperty = propertylist[i].split("=", 1) 981 self['properties'][splitproperty[0]] = splitproperty[1]
982
983 - def opt_patchlevel(self, option):
984 self['patchlevel'] = int(option)
985
986 - def getSynopsis(self):
987 return "Usage: buildbot try [options]"
988
989 - def postOptions(self):
990 opts = loadOptionsFile() 991 if not self['builders']: 992 self['builders'] = opts.get('try_builders', []) 993 if opts.get('try_wait', False): 994 self['wait'] = True 995 if opts.get('try_quiet', False): 996 self['quiet'] = True 997 # get the global 'masterstatus' option if it's set and no master 998 # was specified otherwise 999 if not self['master']: 1000 self['master'] = opts.get('masterstatus', None)
1001
1002 -def doTry(config):
1003 from buildbot.clients import tryclient 1004 t = tryclient.Try(config) 1005 t.run()
1006
1007 -class TryServerOptions(OptionsWithOptionsFile):
1008 optParameters = [ 1009 ["jobdir", None, None, "the jobdir (maildir) for submitting jobs"], 1010 ]
1011 - def getSynopsis(self):
1012 return "Usage: buildbot tryserver [options]"
1013
1014 1015 -def doTryServer(config):
1016 try: 1017 from hashlib import md5 1018 assert md5 1019 except ImportError: 1020 # For Python 2.4 compatibility 1021 import md5 1022 jobdir = os.path.expanduser(config["jobdir"]) 1023 job = sys.stdin.read() 1024 # now do a 'safecat'-style write to jobdir/tmp, then move atomically to 1025 # jobdir/new . Rather than come up with a unique name randomly, I'm just 1026 # going to MD5 the contents and prepend a timestamp. 1027 timestring = "%d" % time.time() 1028 try: 1029 m = md5() 1030 except TypeError: 1031 # For Python 2.4 compatibility 1032 m = md5.new() 1033 m.update(job) 1034 jobhash = m.hexdigest() 1035 fn = "%s-%s" % (timestring, jobhash) 1036 tmpfile = os.path.join(jobdir, "tmp", fn) 1037 newfile = os.path.join(jobdir, "new", fn) 1038 f = open(tmpfile, "w") 1039 f.write(job) 1040 f.close() 1041 os.rename(tmpfile, newfile)
1042
1043 1044 -class CheckConfigOptions(OptionsWithOptionsFile):
1045 optFlags = [ 1046 ['quiet', 'q', "Don't display error messages or tracebacks"], 1047 ] 1048
1049 - def getSynopsis(self):
1050 return "Usage: buildbot checkconfig [configFile]\n" + \ 1051 " If not specified, 'master.cfg' will be used as 'configFile'"
1052
1053 - def parseArgs(self, *args):
1054 if len(args) >= 1: 1055 self['configFile'] = args[0] 1056 else: 1057 self['configFile'] = 'master.cfg'
1058
1059 1060 -def doCheckConfig(config):
1061 from buildbot.scripts.checkconfig import ConfigLoader 1062 quiet = config.get('quiet') 1063 configFileName = config.get('configFile') 1064 1065 if os.path.isdir(configFileName): 1066 os.chdir(configFileName) 1067 cl = ConfigLoader(basedir=configFileName) 1068 else: 1069 cl = ConfigLoader(configFileName=configFileName) 1070 1071 return cl.load(quiet=quiet)
1072
1073 1074 -class UserOptions(OptionsWithOptionsFile):
1075 optParameters = [ 1076 ["master", "m", None, 1077 "Location of the buildmaster's PBListener (host:port)"], 1078 ["username", "u", None, 1079 "Username for PB authentication"], 1080 ["passwd", "p", None, 1081 "Password for PB authentication"], 1082 ["op", None, None, 1083 "User management operation: add, remove, update, get"], 1084 ["bb_username", None, None, 1085 "Username to set for a given user. Only availabe on 'update', " 1086 "and bb_password must be given as well."], 1087 ["bb_password", None, None, 1088 "Password to set for a given user. Only availabe on 'update', " 1089 "and bb_username must be given as well."], 1090 ["ids", None, None, 1091 "User's identifiers, used to find users in 'remove' and 'get' " 1092 "Can be specified multiple times (--ids=id1,id2,id3)"], 1093 ["info", None, None, 1094 "User information in the form: --info=type=value,type=value,.. " 1095 "Used in 'add' and 'update', can be specified multiple times. " 1096 "Note that 'update' requires --info=id:type=value..."] 1097 ] 1098 buildbotOptions = [ 1099 [ 'master', 'master' ], 1100 [ 'user_master', 'master' ], 1101 [ 'user_username', 'username' ], 1102 [ 'user_passwd', 'passwd' ], 1103 ] 1104
1105 - def __init__(self):
1106 OptionsWithOptionsFile.__init__(self) 1107 self['ids'] = [] 1108 self['info'] = []
1109
1110 - def opt_ids(self, option):
1111 id_list = option.split(",") 1112 self['ids'].extend(id_list)
1113
1114 - def opt_info(self, option):
1115 # splits info into type/value dictionary, appends to info 1116 info_list = option.split(",") 1117 info_elem = {} 1118 1119 if len(info_list) == 1 and '=' not in info_list[0]: 1120 info_elem["identifier"] = info_list[0] 1121 self['info'].append(info_elem) 1122 else: 1123 for i in range(0, len(info_list)): 1124 split_info = info_list[i].split("=", 1) 1125 1126 # pull identifier from update --info 1127 if ":" in split_info[0]: 1128 split_id = split_info[0].split(":") 1129 info_elem["identifier"] = split_id[0] 1130 split_info[0] = split_id[1] 1131 1132 info_elem[split_info[0]] = split_info[1] 1133 self['info'].append(info_elem)
1134
1135 - def getSynopsis(self):
1136 return "Usage: buildbot user [options]"
1137 1138 longdesc = """ 1139 Currently implemented types for --info= are:\n 1140 git, svn, hg, cvs, darcs, bzr, email 1141 """
1142
1143 -def users_client(config, runReactor=False):
1144 from buildbot.clients import usersclient 1145 from buildbot.process.users import users # for srcs, encrypt 1146 1147 # accepted attr_types by `buildbot user`, in addition to users.srcs 1148 attr_types = ['identifier', 'email'] 1149 1150 master = config.get('master') 1151 assert master, "you must provide the master location" 1152 try: 1153 master, port = master.split(":") 1154 port = int(port) 1155 except: 1156 raise AssertionError("master must have the form 'hostname:port'") 1157 1158 op = config.get('op') 1159 assert op, "you must specify an operation: add, remove, update, get" 1160 if op not in ['add', 'remove', 'update', 'get']: 1161 raise AssertionError("bad op %r, use 'add', 'remove', 'update', " 1162 "or 'get'" % op) 1163 1164 username = config.get('username') 1165 passwd = config.get('passwd') 1166 assert username and passwd, "A username and password pair must be given" 1167 1168 bb_username = config.get('bb_username') 1169 bb_password = config.get('bb_password') 1170 if bb_username or bb_password: 1171 if op != 'update': 1172 raise AssertionError("bb_username and bb_password only work " 1173 "with update") 1174 if not bb_username or not bb_password: 1175 raise AssertionError("Must specify both bb_username and " 1176 "bb_password or neither.") 1177 1178 bb_password = users.encrypt(bb_password) 1179 1180 # check op and proper args 1181 info = config.get('info') 1182 ids = config.get('ids') 1183 1184 # check for erroneous args 1185 if not info and not ids: 1186 raise AssertionError("must specify either --ids or --info") 1187 if info and ids: 1188 raise AssertionError("cannot use both --ids and --info, use " 1189 "--ids for 'remove' and 'get', --info " 1190 "for 'add' and 'update'") 1191 1192 if op == 'add' or op == 'update': 1193 if ids: 1194 raise AssertionError("cannot use --ids with 'add' or 'update'") 1195 if op == 'update': 1196 for user in info: 1197 if 'identifier' not in user: 1198 raise ValueError("no ids found in update info, use: " 1199 "--info=id:type=value,type=value,..") 1200 if op == 'add': 1201 for user in info: 1202 if 'identifier' in user: 1203 raise ValueError("id found in add info, use: " 1204 "--info=type=value,type=value,..") 1205 if op == 'remove' or op == 'get': 1206 if info: 1207 raise AssertionError("cannot use --info with 'remove' or 'get'") 1208 1209 # find identifier if op == add 1210 if info: 1211 # check for valid types 1212 for user in info: 1213 for attr_type in user: 1214 if attr_type not in users.srcs + attr_types: 1215 raise ValueError("Type not a valid attr_type, must be in: " 1216 "%r" % (users.srcs + attr_types)) 1217 1218 if op == 'add': 1219 user['identifier'] = user.values()[0] 1220 1221 uc = usersclient.UsersClient(master, username, passwd, port) 1222 d = uc.send(op, bb_username, bb_password, ids, info) 1223 1224 if runReactor: 1225 from twisted.internet import reactor 1226 status = [True] 1227 def printSuccess(res): 1228 print res
1229 def failed(f): 1230 status[0] = False 1231 print "user op NOT sent - something went wrong: " + str(f) 1232 d.addCallbacks(printSuccess, failed) 1233 d.addBoth(lambda _ : reactor.stop()) 1234 reactor.run() 1235 return status[0] 1236 return d 1237
1238 -class Options(usage.Options):
1239 synopsis = "Usage: buildbot <command> [command options]" 1240 1241 subCommands = [ 1242 # the following are all admin commands 1243 ['create-master', None, MasterOptions, 1244 "Create and populate a directory for a new buildmaster"], 1245 ['upgrade-master', None, UpgradeMasterOptions, 1246 "Upgrade an existing buildmaster directory for the current version"], 1247 ['start', None, StartOptions, "Start a buildmaster"], 1248 ['stop', None, StopOptions, "Stop a buildmaster"], 1249 ['restart', None, RestartOptions, 1250 "Restart a buildmaster"], 1251 1252 ['reconfig', None, ReconfigOptions, 1253 "SIGHUP a buildmaster to make it re-read the config file"], 1254 ['sighup', None, ReconfigOptions, 1255 "SIGHUP a buildmaster to make it re-read the config file"], 1256 1257 ['sendchange', None, SendChangeOptions, 1258 "Send a change to the buildmaster"], 1259 1260 ['debugclient', None, DebugClientOptions, 1261 "Launch a small debug panel GUI"], 1262 1263 ['statuslog', None, StatusLogOptions, 1264 "Emit current builder status to stdout"], 1265 ['statusgui', None, StatusGuiOptions, 1266 "Display a small window showing current builder status"], 1267 1268 #['force', None, ForceOptions, "Run a build"], 1269 ['try', None, TryOptions, "Run a build with your local changes"], 1270 1271 ['tryserver', None, TryServerOptions, 1272 "buildmaster-side 'try' support function, not for users"], 1273 1274 ['checkconfig', None, CheckConfigOptions, 1275 "test the validity of a master.cfg config file"], 1276 1277 ['user', None, UserOptions, 1278 "Manage users in buildbot's database"] 1279 1280 # TODO: 'watch' 1281 ] 1282
1283 - def opt_version(self):
1284 import buildbot 1285 print "Buildbot version: %s" % buildbot.version 1286 usage.Options.opt_version(self)
1287
1288 - def opt_verbose(self):
1289 from twisted.python import log 1290 log.startLogging(sys.stderr)
1291
1292 - def postOptions(self):
1293 if not hasattr(self, 'subOptions'): 1294 raise usage.UsageError("must specify a command")
1295
1296 1297 -def run():
1298 config = Options() 1299 try: 1300 config.parseOptions() 1301 except usage.error, e: 1302 print "%s: %s" % (sys.argv[0], e) 1303 print 1304 c = getattr(config, 'subOptions', config) 1305 print str(c) 1306 sys.exit(1) 1307 1308 command = config.subCommand 1309 so = config.subOptions 1310 1311 if command == "create-master": 1312 createMaster(so) 1313 elif command == "upgrade-master": 1314 upgradeMaster(so) 1315 elif command == "start": 1316 from buildbot.scripts.startup import start 1317 1318 if not isBuildmasterDir(so['basedir']): 1319 print "not a buildmaster directory" 1320 sys.exit(1) 1321 1322 start(so) 1323 elif command == "stop": 1324 try: 1325 stop(so, wait=True) 1326 except BuildbotNotRunningError: 1327 if not so['quiet']: 1328 print "buildmaster not running" 1329 sys.exit(0) 1330 1331 elif command == "restart": 1332 restart(so) 1333 elif command == "reconfig" or command == "sighup": 1334 from buildbot.scripts.reconfig import Reconfigurator 1335 Reconfigurator().run(so) 1336 elif command == "sendchange": 1337 if not sendchange(so, True): 1338 sys.exit(1) 1339 elif command == "debugclient": 1340 debugclient(so) 1341 elif command == "statuslog": 1342 statuslog(so) 1343 elif command == "statusgui": 1344 statusgui(so) 1345 elif command == "try": 1346 doTry(so) 1347 elif command == "tryserver": 1348 doTryServer(so) 1349 elif command == "checkconfig": 1350 if not doCheckConfig(so): 1351 sys.exit(1) 1352 elif command == "user": 1353 if not users_client(so, True): 1354 sys.exit(1) 1355 sys.exit(0)
1356