buildbot.schedulers.forcesched

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 import traceback 17 import re 18 from twisted.internet import defer 19 import email.utils as email_utils 20 21 from buildbot.process.properties import Properties 22 from buildbot.schedulers import base

23 24 -class ValidationError(ValueError):

25 pass

26 27 DefaultField = object() # sentinel object to signal default behavior

28 29 -class BaseParameter(object):

30 """ 31 BaseParameter provides a base implementation for property customization 32 """ 33 name = "" 34 parentName = None 35 label = "" 36 type = [] 37 default = "" 38 required = False 39 multiple = False 40 regex = None 41 debug=True 42 hide = False 43 44 @property

45 - def fullName(self):

46 """A full name, intended to uniquely identify a parameter""" 47 # join with '_' if both are set 48 if self.parentName and self.name: 49 return self.parentName+'_'+self.name 50 # otherwise just use the one that is set 51 # (this allows empty name for "anonymous nests") 52 return self.name or self.parentName

53

54 - def setParent(self, parent):

55 self.parentName = parent.fullName if parent else None

56

57 - def __init__(self, name, label=None, regex=None, **kw):

58 """ 59 @param name: the name of the field, used during posting values 60 back to the scheduler. This is not necessarily a UI value, 61 and there may be restrictions on the characters allowed for 62 this value. For example, HTML would require this field to 63 avoid spaces and other punctuation ('-', '.', and '_' allowed) 64 @type name: unicode 65 66 @param label: (optional) the name of the field, used for UI display. 67 @type label: unicode or None (to use 'name') 68 69 @param regex: (optional) regex to validate the value with. Not used by 70 all subclasses 71 @type regex: unicode or regex 72 """ 73 74 self.name = name 75 self.label = name if label is None else label 76 if regex: 77 self.regex = re.compile(regex) 78 # all other properties are generically passed via **kw 79 self.__dict__.update(kw)

80

81 - def getFromKwargs(self, kwargs):

82 """Simple customization point for child classes that do not need the other 83 parameters supplied to updateFromKwargs. Return the value for the property 84 named 'self.name'. 85 86 The default implementation converts from a list of items, validates using 87 the optional regex field and calls 'parse_from_args' for the final conversion. 88 """ 89 args = kwargs.get(self.fullName, []) 90 if len(args) == 0: 91 if self.required: 92 raise ValidationError("'%s' needs to be specified" % (self.label)) 93 if self.multiple: 94 args = self.default 95 else: 96 args = [self.default] 97 98 if self.regex: 99 for arg in args: 100 if not self.regex.match(arg): 101 raise ValidationError("%s:'%s' does not match pattern '%s'" 102 % (self.label, arg, self.regex.pattern)) 103 104 try: 105 arg = self.parse_from_args(args) 106 except Exception, e: 107 # an exception will just display an alert in the web UI 108 # also log the exception 109 if self.debug: 110 traceback.print_exc() 111 raise e 112 if arg is None: 113 raise ValidationError("need %s: no default provided by config" 114 % (self.fullName,)) 115 return arg

116

117 - def updateFromKwargs(self, properties, kwargs, **unused):

118 """Primary entry point to turn 'kwargs' into 'properties'""" 119 properties[self.name] = self.getFromKwargs(kwargs)

120

121 - def parse_from_args(self, l):

122 """Secondary customization point, called from getFromKwargs to turn 123 a validated value into a single property value""" 124 if self.multiple: 125 return map(self.parse_from_arg, l) 126 else: 127 return self.parse_from_arg(l[0])

128

129 - def parse_from_arg(self, s):

130 return s

131

132 133 -class FixedParameter(BaseParameter):

134 """A fixed parameter that cannot be modified by the user.""" 135 type = ["fixed"] 136 hide = True 137 default = "" 138

139 - def parse_from_args(self, l):

140 return self.default

141

142 143 -class StringParameter(BaseParameter):

144 """A simple string parameter""" 145 type = ["text"] 146 size = 10 147

148 - def parse_from_arg(self, s):

149 return s

150

151 152 -class TextParameter(StringParameter):

153 """A generic string parameter that may span multiple lines""" 154 type = ["textarea"] 155 cols = 80 156 rows = 20 157

158 - def value_to_text(self, value):

159 return str(value)

160

161 162 -class IntParameter(StringParameter):

163 """An integer parameter""" 164 type = ["int"] 165 166 parse_from_arg = int # will throw an exception if parse fail

167

168 169 -class BooleanParameter(BaseParameter):

170 """A boolean parameter""" 171 type = ["bool"] 172

173 - def getFromKwargs(self, kwargs):

174 return kwargs.get(self.fullName, None) == [True]

175

176 177 -class UserNameParameter(StringParameter):

178 """A username parameter to supply the 'owner' of a build""" 179 type = ["text"] 180 default = "" 181 size = 30 182 need_email = True 183

184 - def __init__(self, name="username", label="Your name:", **kw):

185 BaseParameter.__init__(self, name, label, **kw)

186

187 - def parse_from_arg(self, s):

188 if not s and not self.required: 189 return s 190 if self.need_email: 191 e = email_utils.parseaddr(s) 192 if e[0]=='' or e[1] == '': 193 raise ValidationError("%s: please fill in email address in the " 194 "form 'User <email@email.com>'" % (self.name,)) 195 return s

196

197 198 -class ChoiceStringParameter(BaseParameter):

199 """A list of strings, allowing the selection of one of the predefined values. 200 The 'strict' parameter controls whether values outside the predefined list 201 of choices are allowed""" 202 type = ["list"] 203 choices = [] 204 strict = True 205

206 - def parse_from_arg(self, s):

207 if self.strict and not s in self.choices: 208 raise ValidationError("'%s' does not belongs to list of available choices '%s'"%(s, self.choices)) 209 return s

210

211 212 213 -class InheritBuildParameter(ChoiceStringParameter):

214 """A parameter that takes its values from another build""" 215 type = ChoiceStringParameter.type + ["inherit"] 216 name = "inherit" 217 compatible_builds = None 218

219 - def getFromKwargs(self, kwargs):

220 raise ValidationError("InheritBuildParameter can only be used by properties")

221

222 - def updateFromKwargs(self, master, properties, changes, kwargs, **unused):

223 arg = kwargs.get(self.fullName, [""])[0] 224 splitted_arg = arg.split(" ")[0].split("/") 225 if len(splitted_arg) != 2: 226 raise ValidationError("bad build: %s"%(arg)) 227 builder, num = splitted_arg 228 builder_status = master.status.getBuilder(builder) 229 if not builder_status: 230 raise ValidationError("unknown builder: %s in %s"%(builder, arg)) 231 b = builder_status.getBuild(int(num)) 232 if not b: 233 raise ValidationError("unknown build: %d in %s"%(num, arg)) 234 props = {self.name:(arg.split(" ")[0])} 235 for name, value, source in b.getProperties().asList(): 236 if source == "Force Build Form": 237 if name == "owner": 238 name = "orig_owner" 239 props[name] = value 240 properties.update(props) 241 changes.extend(b.changes)

242

243 244 -class NestedParameter(BaseParameter):

245 """A 'parent' parameter for a set of related parameters. This provices a 246 logical grouping for the child parameters. 247 248 Typically, the 'fullName' of the child parameters mix in the parent's 249 'fullName'. This allows for a field to appear multiple times in a form 250 (for example, two codebases each have a 'branch' field). 251 252 If the 'name' of the parent is the empty string, then the parent's name 253 does not mix in with the child 'fullName'. This is useful when a field 254 will not appear multiple time in a scheduler but the logical grouping is 255 helpful. 256 257 The result of a NestedParameter is typically a dictionary, with the key/value 258 being the name/value of the children. 259 """ 260 type = ['nested'] 261 fields = None 262

263 - def __init__(self, name, fields, **kwargs):

264 BaseParameter.__init__(self, fields=fields, name=name, **kwargs) 265 266 # fix up the child nodes with the parent (use None for now): 267 self.setParent(None)

268

269 - def setParent(self, parent):

270 BaseParameter.setParent(self, parent) 271 for field in self.fields: 272 field.setParent(self)

273

274 - def collectChildProperties(self, kwargs, properties, **kw):

275 """Collapse the child values into a dictionary. This is intended to be 276 called by child classes to fix up the fullName->name conversions.""" 277 278 childProperties = {} 279 for field in self.fields: 280 field.updateFromKwargs(kwargs=kwargs, 281 properties=childProperties, 282 **kw) 283 284 kwargs[self.fullName] = childProperties

285

286 - def updateFromKwargs(self, kwargs, properties, **kw):

287 """By default, the child values will be collapsed into a dictionary. If 288 the parent is anonymous, this dictionary is the top-level properties.""" 289 self.collectChildProperties(kwargs=kwargs, properties=properties, **kw) 290 291 # default behavior is to set a property 292 # -- use setdefault+update in order to collapse 'anonymous' nested 293 # parameters correctly 294 if self.name: 295 d = properties.setdefault(self.name, {}) 296 else: 297 # if there's no name, collapse this nest all the way 298 d = properties 299 d.update(kwargs[self.fullName])

300

301 -class AnyPropertyParameter(NestedParameter):

302 """A generic property parameter, where both the name and value of the property 303 must be given.""" 304 type = NestedParameter.type + ["any"] 305

306 - def __init__(self, name, **kw):

307 fields = [ 308 StringParameter(name='name', label="Name:"), 309 StringParameter(name='value', label="Value:"), 310 ] 311 NestedParameter.__init__(self, name, label='', fields=fields, **kw)

312

313 - def getFromKwargs(self, kwargs):

314 raise ValidationError("AnyPropertyParameter can only be used by properties")

315

316 - def updateFromKwargs(self, master, properties, kwargs, **kw):

317 self.collectChildProperties(master=master, 318 properties=properties, 319 kwargs=kwargs, 320 **kw) 321 322 pname = kwargs[self.fullName].get("name", "") 323 pvalue = kwargs[self.fullName].get("value", "") 324 if not pname: 325 return 326 327 validation = master.config.validation 328 pname_validate = validation['property_name'] 329 pval_validate = validation['property_value'] 330 331 if not pname_validate.match(pname) \ 332 or not pval_validate.match(pvalue): 333 raise ValidationError("bad property name='%s', value='%s'" % (pname, pvalue)) 334 properties[pname] = pvalue

335

336 337 -class CodebaseParameter(NestedParameter):

338 """A parameter whose result is a codebase specification instead of a property""" 339 type = NestedParameter.type + ["codebase"] 340 codebase = '' 341

342 - def __init__(self, 343 codebase, 344 name=None, 345 label=None, 346 347 branch=DefaultField, 348 revision=DefaultField, 349 repository=DefaultField, 350 project=DefaultField, 351 352 **kwargs):

353 """ 354 A set of properties that will be used to generate a codebase dictionary. 355 356 The branch/revision/repository/project should each be a parameter that 357 will map to the corresponding value in the sourcestamp. Use None to disable 358 the field. 359 360 @param codebase: name of the codebase; used as key for the sourcestamp set 361 @type codebase: unicode 362 363 @param name: optional override for the name-currying for the subfields 364 @type codebase: unicode 365 366 @param label: optional override for the label for this set of parameters 367 @type codebase: unicode 368 """ 369 370 name = name or codebase 371 if label is None and codebase: 372 label = "Codebase: " + codebase 373 374 if branch is DefaultField: 375 branch = StringParameter(name='branch', label="Branch:") 376 if revision is DefaultField: 377 revision = StringParameter(name='revision', label="Revision:") 378 if repository is DefaultField: 379 repository = StringParameter(name='repository', label="Repository:") 380 if project is DefaultField: 381 project = StringParameter(name='project', label="Project:") 382 383 fields = filter(None, [branch, revision, repository, project]) 384 385 NestedParameter.__init__(self, name=name, label=label, 386 codebase=codebase, 387 fields=fields, **kwargs)

388

389 - def createSourcestamp(self, properties, kwargs):

390 # default, just return the things we put together 391 return kwargs.get(self.fullName, {})

392

393 - def updateFromKwargs(self, sourcestamps, kwargs, properties, **kw):

394 self.collectChildProperties(sourcestamps=sourcestamps, 395 properties=properties, 396 kwargs=kwargs, 397 **kw) 398 399 # convert the "property" to a sourcestamp 400 ss = self.createSourcestamp(properties, kwargs) 401 if ss is not None: 402 sourcestamps[self.codebase] = ss

403

404 405 -class ForceScheduler(base.BaseScheduler):

406 """ 407 ForceScheduler implements the backend for a UI to allow customization of 408 builds. For example, a web form be populated to trigger a build. 409 """ 410 compare_attrs = ( 'name', 'builderNames', 411 'reason', 'username', 412 'forcedProperties' ) 413

414 - def __init__(self, name, builderNames, 415 username=UserNameParameter(), 416 reason=StringParameter(name="reason", default="force build", length=20), 417 418 codebases=None, 419 420 properties=[ 421 NestedParameter(name='', fields=[ 422 AnyPropertyParameter("property1"), 423 AnyPropertyParameter("property2"), 424 AnyPropertyParameter("property3"), 425 AnyPropertyParameter("property4"), 426 ]) 427 ], 428 429 # deprecated; use 'codebase' instead 430 branch=None, 431 revision=None, 432 repository=None, 433 project=None 434 ):

435 """ 436 Initialize a ForceScheduler. 437 438 The UI will provide a set of fields to the user; these fields are 439 driven by a corresponding child class of BaseParameter. 440 441 Use NestedParameter to provide logical groupings for parameters. 442 443 The branch/revision/repository/project fields are deprecated and 444 provided only for backwards compatibility. Using a Codebase(name='') 445 will give the equivalent behavior. 446 447 @param name: name of this scheduler (used as a key for state) 448 @type name: unicode 449 450 @param builderNames: list of builders this scheduler may start 451 @type builderNames: list of unicode 452 453 @param username: the "owner" for a build (may not be shown depending 454 on the Auth configuration for the master) 455 @type reason: BaseParameter or None (to disable this field) 456 457 @param reason: the "reason" for a build 458 @type reason: BaseParameter or None (to disable this field) 459 460 @param codebases: the codebases for a build 461 @type codebases: list of string's or CodebaseParameter's; 462 None will generate a default, but [] will 463 remove all codebases 464 465 @param properties: extra properties to configure the build 466 @type properties: list of BaseParameter's 467 """ 468 469 self.reason = reason 470 self.username = username 471 472 self.forcedProperties = [] 473 474 if any((branch, revision, repository, project)): 475 if codebases: 476 raise ValidationError("Must either specify 'codebases' or the 'branch/revision/repository/project' parameters") 477 478 codebases = [ 479 CodebaseParameter(codebase='', 480 branch=branch or DefaultField, 481 revision=revision or DefaultField, 482 repository=repository or DefaultField, 483 project=project or DefaultField, 484 ) 485 ] 486 487 # Use the default single codebase form if none are provided 488 if codebases is None: 489 codebases =[CodebaseParameter(codebase='')] 490 elif not codebases: 491 raise ValidationError("'codebases' cannot be empty; use CodebaseParameter(codebase='', hide=True) if needed") 492 493 codebase_dict = {} 494 for codebase in codebases: 495 if isinstance(codebase, basestring): 496 codebase = CodebaseParameter(codebase=codebase) 497 elif not isinstance(codebase, CodebaseParameter): 498 raise ValidationError("'codebases' must be a list of strings or CodebaseParameter objects") 499 500 self.forcedProperties.append(codebase) 501 codebase_dict[codebase.codebase] = dict(branch='',repository='',revision='') 502 503 base.BaseScheduler.__init__(self, 504 name=name, 505 builderNames=builderNames, 506 properties={}, 507 codebases=codebase_dict) 508 509 if properties: 510 self.forcedProperties.extend(properties) 511 512 # this is used to simplify the template 513 self.all_fields = [ NestedParameter(name='', fields=[username, reason]) ] 514 self.all_fields.extend(self.forcedProperties)

515

516 - def startService(self):

517 pass

518

519 - def stopService(self):

520 pass

521 522 @defer.inlineCallbacks

523 - def gatherPropertiesAndChanges(self, **kwargs):

524 properties = {} 525 changeids = [] 526 sourcestamps = {} 527 528 for param in self.forcedProperties: 529 yield defer.maybeDeferred(param.updateFromKwargs, 530 master=self.master, 531 properties=properties, 532 changes=changeids, 533 sourcestamps=sourcestamps, 534 kwargs=kwargs) 535 536 changeids = map(lambda a: type(a)==int and a or a.number, changeids) 537 538 real_properties = Properties() 539 for pname, pvalue in properties.items(): 540 real_properties.setProperty(pname, pvalue, "Force Build Form") 541 542 defer.returnValue((real_properties, changeids, sourcestamps))

543 544 @defer.inlineCallbacks

545 - def force(self, owner, builder_name, **kwargs):

546 """ 547 We check the parameters, and launch the build, if everything is correct 548 """ 549 if not builder_name in self.builderNames: 550 # in the case of buildAll, this method will be called several times 551 # for all the builders 552 # we just do nothing on a builder that is not in our builderNames 553 defer.returnValue(None) 554 return 555 556 # Currently the validation code expects all kwargs to be lists 557 # I don't want to refactor that now so much sure we comply... 558 kwargs = dict((k, [v]) if not isinstance(v, list) else (k,v) for k,v in kwargs.items()) 559 560 # probably need to clean that out later as the IProperty is already a 561 # validation mechanism 562 563 reason = self.reason.getFromKwargs(kwargs) 564 if owner is None: 565 owner = self.username.getFromKwargs(kwargs) 566 567 properties, changeids, sourcestamps = yield self.gatherPropertiesAndChanges(**kwargs) 568 569 properties.setProperty("reason", reason, "Force Build Form") 570 properties.setProperty("owner", owner, "Force Build Form") 571 572 r = ("A build was forced by '%s': %s" % (owner, reason)) 573 574 # everything is validated, we can create our source stamp, and buildrequest 575 res = yield self.addBuildsetForSourceStampSetDetails( 576 reason = r, 577 sourcestamps = sourcestamps, 578 properties = properties, 579 builderNames = [ builder_name ], 580 ) 581 582 defer.returnValue(res)

583

Source Code for Module buildbot.schedulers.forcesched