1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 import os
18 import signal
19 import socket
20
21 from zope.interface import implements
22 from twisted.python import log, components, failure
23 from twisted.internet import defer, reactor, task
24 from twisted.application import service
25
26 import buildbot
27 import buildbot.pbmanager
28 from buildbot.util import subscription, epoch2datetime
29 from buildbot.status.master import Status
30 from buildbot.changes import changes
31 from buildbot.changes.manager import ChangeManager
32 from buildbot import interfaces
33 from buildbot.process.builder import BuilderControl
34 from buildbot.db import connector
35 from buildbot.schedulers.manager import SchedulerManager
36 from buildbot.process.botmaster import BotMaster
37 from buildbot.process import debug
38 from buildbot.process import metrics
39 from buildbot.process import cache
40 from buildbot.process.users import users
41 from buildbot.process.users.manager import UserManagerManager
42 from buildbot.status.results import SUCCESS, WARNINGS, FAILURE
43 from buildbot import monkeypatches
44 from buildbot import config
50 self.rotateLength = 1 * 1000 * 1000
51 self.maxRotatedFiles = 10
52
53 -class BuildMaster(config.ReconfigurableServiceMixin, service.MultiService):
215
216
218 if self.db_loop:
219 self.db_loop.stop()
220 self.db_loop = None
221
222
224
225
226 if self.reconfig_active:
227 log.msg("reconfig already active; will reconfig again after")
228 self.reconfig_requested = True
229 return
230
231 self.reconfig_active = reactor.seconds()
232 metrics.MetricCountEvent.log("loaded_config", 1)
233
234
235
236 self.reconfig_notifier = task.LoopingCall(lambda :
237 log.msg("reconfig is ongoing for %d s" %
238 (reactor.seconds() - self.reconfig_active)))
239 self.reconfig_notifier.start(10, now=False)
240
241 timer = metrics.Timer("BuildMaster.reconfig")
242 timer.start()
243
244 d = self.doReconfig()
245
246 @d.addBoth
247 def cleanup(res):
248 timer.stop()
249 self.reconfig_notifier.stop()
250 self.reconfig_notifier = None
251 self.reconfig_active = False
252 if self.reconfig_requested:
253 self.reconfig_requested = False
254 self.reconfig()
255 return res
256
257 d.addErrback(log.err, 'while reconfiguring')
258
259 return d
260
261
262 @defer.deferredGenerator
264 log.msg("beginning configuration update")
265 changes_made = False
266 failed = False
267 try:
268 new_config = config.MasterConfig.loadConfig(self.basedir,
269 self.configFileName)
270 changes_made = True
271 self.config = new_config
272 wfd = defer.waitForDeferred(
273 self.reconfigService(new_config))
274 yield wfd
275 wfd.getResult()
276
277 except config.ConfigErrors, e:
278 for msg in e.errors:
279 log.msg(msg)
280 failed = True
281
282 except:
283 log.err(failure.Failure(), 'during reconfig:')
284 failed = True
285
286 if failed:
287 if changes_made:
288 log.msg("WARNING: reconfig partially applied; master "
289 "may malfunction")
290 else:
291 log.msg("reconfig aborted without making any changes")
292 else:
293 log.msg("configuration update complete")
294
295
297 if self.config.db['db_url'] != new_config.db['db_url']:
298 config.error(
299 "Cannot change c['db']['db_url'] after the master has started",
300 )
301
302
303 if (self.config.db['db_poll_interval']
304 != new_config.db['db_poll_interval']):
305 if self.db_loop:
306 self.db_loop.stop()
307 self.db_loop = None
308 poll_interval = new_config.db['db_poll_interval']
309 if poll_interval:
310 self.db_loop = task.LoopingCall(self.pollDatabase)
311 self.db_loop.start(poll_interval, now=False)
312
313 return config.ReconfigurableServiceMixin.reconfigService(self,
314 new_config)
315
316
317
318
320 return list(self.scheduler_manager)
321
323 """
324 @rtype: L{buildbot.status.builder.Status}
325 """
326 return self.status
327
329 """
330 Return the obejct id for this master, for associating state with the
331 master.
332
333 @returns: ID, via Deferred
334 """
335
336 if self._object_id is not None:
337 return defer.succeed(self._object_id)
338
339
340
341 try:
342 hostname = os.uname()[1]
343 except AttributeError:
344 hostname = socket.getfqdn()
345 master_name = "%s:%s" % (hostname, os.path.abspath(self.basedir))
346
347 d = self.db.state.getObjectId(master_name,
348 "buildbot.master.BuildMaster")
349 def keep(id):
350 self._object_id = id
351 return id
352 d.addCallback(keep)
353 return d
354
355
356
357
358 - def addChange(self, who=None, files=None, comments=None, author=None,
359 isdir=None, is_dir=None, revision=None, when=None,
360 when_timestamp=None, branch=None, category=None, revlink='',
361 properties={}, repository='', project='', src=None):
362 """
363 Add a change to the buildmaster and act on it.
364
365 This is a wrapper around L{ChangesConnectorComponent.addChange} which
366 also acts on the resulting change and returns a L{Change} instance.
367
368 Note that all parameters are keyword arguments, although C{who},
369 C{files}, and C{comments} can be specified positionally for
370 backward-compatibility.
371
372 @param author: the author of this change
373 @type author: unicode string
374
375 @param who: deprecated name for C{author}
376
377 @param files: a list of filenames that were changed
378 @type branch: list of unicode strings
379
380 @param comments: user comments on the change
381 @type branch: unicode string
382
383 @param is_dir: deprecated
384
385 @param isdir: deprecated name for C{is_dir}
386
387 @param revision: the revision identifier for this change
388 @type revision: unicode string
389
390 @param when_timestamp: when this change occurred, or the current time
391 if None
392 @type when_timestamp: datetime instance or None
393
394 @param when: deprecated name and type for C{when_timestamp}
395 @type when: integer (UNIX epoch time) or None
396
397 @param branch: the branch on which this change took place
398 @type branch: unicode string
399
400 @param category: category for this change (arbitrary use by Buildbot
401 users)
402 @type category: unicode string
403
404 @param revlink: link to a web view of this revision
405 @type revlink: unicode string
406
407 @param properties: properties to set on this change
408 @type properties: dictionary with string keys and simple values
409 (JSON-able). Note that the property source is I{not} included
410 in this dictionary.
411
412 @param repository: the repository in which this change took place
413 @type repository: unicode string
414
415 @param project: the project this change is a part of
416 @type project: unicode string
417
418 @param src: source of the change (vcs or other)
419 @type src: string
420
421 @returns: L{Change} instance via Deferred
422 """
423 metrics.MetricCountEvent.log("added_changes", 1)
424
425
426 def handle_deprec(oldname, old, newname, new, default=None,
427 converter = lambda x:x):
428 if old is not None:
429 if new is None:
430 log.msg("WARNING: change source is using deprecated "
431 "addChange parameter '%s'" % oldname)
432 return converter(old)
433 raise TypeError("Cannot provide '%s' and '%s' to addChange"
434 % (oldname, newname))
435 if new is None:
436 new = default
437 return new
438
439 author = handle_deprec("who", who, "author", author)
440 is_dir = handle_deprec("isdir", isdir, "is_dir", is_dir,
441 default=0)
442 when_timestamp = handle_deprec("when", when,
443 "when_timestamp", when_timestamp,
444 converter=epoch2datetime)
445
446
447 for n in properties:
448 properties[n] = (properties[n], 'Change')
449
450 d = defer.succeed(None)
451 if src:
452
453 d.addCallback(lambda _ : users.createUserObject(self, author, src))
454
455
456 d.addCallback(lambda uid :
457 self.db.changes.addChange(author=author, files=files,
458 comments=comments, is_dir=is_dir,
459 revision=revision,
460 when_timestamp=when_timestamp,
461 branch=branch, category=category,
462 revlink=revlink, properties=properties,
463 repository=repository, project=project,
464 uid=uid))
465
466
467 d.addCallback(lambda changeid :
468 self.db.changes.getChange(changeid))
469 d.addCallback(lambda chdict :
470 changes.Change.fromChdict(self, chdict))
471
472 def notify(change):
473 msg = u"added change %s to database" % change
474 log.msg(msg.encode('utf-8', 'replace'))
475
476 if not self.config.db['db_poll_interval']:
477 self._change_subs.deliver(change)
478 return change
479 d.addCallback(notify)
480 return d
481
483 """
484 Request that C{callback} be called with each Change object added to the
485 cluster.
486
487 Note: this method will go away in 0.9.x
488 """
489 return self._change_subs.subscribe(callback)
490
492 """
493 Add a buildset to the buildmaster and act on it. Interface is
494 identical to
495 L{buildbot.db.buildsets.BuildsetConnectorComponent.addBuildset},
496 including returning a Deferred, but also potentially triggers the
497 resulting builds.
498 """
499 d = self.db.buildsets.addBuildset(**kwargs)
500 def notify((bsid,brids)):
501 log.msg("added buildset %d to database" % bsid)
502
503 self._new_buildset_subs.deliver(bsid=bsid, **kwargs)
504
505 if not self.config.db['db_poll_interval']:
506 for bn, brid in brids.iteritems():
507 self.buildRequestAdded(bsid=bsid, brid=brid,
508 buildername=bn)
509 return (bsid,brids)
510 d.addCallback(notify)
511 return d
512
514 """
515 Request that C{callback(bsid=bsid, ssid=ssid, reason=reason,
516 properties=properties, builderNames=builderNames,
517 external_idstring=external_idstring)} be called whenever a buildset is
518 added. Properties is a dictionary as expected for
519 L{BuildsetsConnectorComponent.addBuildset}.
520
521 Note that this only works for buildsets added on this master.
522
523 Note: this method will go away in 0.9.x
524 """
525 return self._new_buildset_subs.subscribe(callback)
526
527 @defer.deferredGenerator
529 """
530 Instructs the master to check whether the buildset is complete,
531 and notify appropriately if it is.
532
533 Note that buildset completions are only reported on the master
534 on which the last build request completes.
535 """
536 wfd = defer.waitForDeferred(
537 self.db.buildrequests.getBuildRequests(bsid=bsid, complete=False))
538 yield wfd
539 brdicts = wfd.getResult()
540
541
542 if brdicts:
543 return
544
545 wfd = defer.waitForDeferred(
546 self.db.buildrequests.getBuildRequests(bsid=bsid))
547 yield wfd
548 brdicts = wfd.getResult()
549
550
551 cumulative_results = SUCCESS
552 for brdict in brdicts:
553 if brdict['results'] not in (SUCCESS, WARNINGS):
554 cumulative_results = FAILURE
555
556
557 wfd = defer.waitForDeferred(
558 self.db.buildsets.completeBuildset(bsid, cumulative_results))
559 yield wfd
560 wfd.getResult()
561
562
563 self._buildsetComplete(bsid, cumulative_results)
564
567
569 """
570 Request that C{callback(bsid, result)} be called whenever a
571 buildset is complete.
572
573 Note: this method will go away in 0.9.x
574 """
575 return self._complete_buildset_subs.subscribe(callback)
576
578 """
579 Notifies the master that a build request is available to be claimed;
580 this may be a brand new build request, or a build request that was
581 previously claimed and unclaimed through a timeout or other calamity.
582
583 @param bsid: containing buildset id
584 @param brid: buildrequest ID
585 @param buildername: builder named by the build request
586 """
587 self._new_buildrequest_subs.deliver(
588 dict(bsid=bsid, brid=brid, buildername=buildername))
589
591 """
592 Request that C{callback} be invoked with a dictionary with keys C{brid}
593 (the build request id), C{bsid} (buildset id) and C{buildername}
594 whenever a new build request is added to the database. Note that, due
595 to the delayed nature of subscriptions, the build request may already
596 be claimed by the time C{callback} is invoked.
597
598 Note: this method will go away in 0.9.x
599 """
600 return self._new_buildrequest_subs.subscribe(callback)
601
602
603
604
618
619 _last_processed_change = None
620 @defer.deferredGenerator
691
692 _last_unclaimed_brids_set = None
693 _last_claim_cleanup = 0
694 @defer.deferredGenerator
744
745
746
752 d.addCallback(get)
753 return d
754
756 "private wrapper around C{self.db.state.setState}"
757 d = self.getObjectId()
758 def set(objectid):
759 return self.db.state.setState(objectid, name, value)
760 d.addCallback(set)
761 return d
762
778
779 components.registerAdapter(Control, BuildMaster, interfaces.IControl)
780