buildbot.status.master

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 from __future__ import with_statement 17 18 import os, urllib 19 from cPickle import load 20 from twisted.python import log 21 from twisted.persisted import styles 22 from twisted.internet import defer 23 from twisted.application import service 24 from zope.interface import implements 25 from buildbot import config, interfaces, util 26 from buildbot.util import bbcollections 27 from buildbot.util.eventual import eventually 28 from buildbot.changes import changes 29 from buildbot.status import buildset, builder, buildrequest

30 31 -class Status(config.ReconfigurableServiceMixin, service.MultiService):

32 implements(interfaces.IStatus) 33

34 - def __init__(self, master):

35 service.MultiService.__init__(self) 36 self.master = master 37 self.botmaster = master.botmaster 38 self.basedir = master.basedir 39 self.watchers = [] 40 # No default limit to the log size 41 self.logMaxSize = None 42 43 self._builder_observers = bbcollections.KeyedSets() 44 self._buildreq_observers = bbcollections.KeyedSets() 45 self._buildset_finished_waiters = bbcollections.KeyedSets() 46 self._buildset_completion_sub = None 47 self._buildset_sub = None 48 self._build_request_sub = None 49 self._change_sub = None

50 51 # service management 52

53 - def startService(self):

54 # subscribe to the things we need to know about 55 self._buildset_completion_sub = \ 56 self.master.subscribeToBuildsetCompletions( 57 self._buildsetCompletionCallback) 58 self._buildset_sub = \ 59 self.master.subscribeToBuildsets( 60 self._buildsetCallback) 61 self._build_request_sub = \ 62 self.master.subscribeToBuildRequests( 63 self._buildRequestCallback) 64 self._change_sub = \ 65 self.master.subscribeToChanges( 66 self.changeAdded) 67 68 return service.MultiService.startService(self)

69 70 @defer.inlineCallbacks

71 - def reconfigService(self, new_config):

72 # remove the old listeners, then add the new 73 for sr in list(self): 74 yield defer.maybeDeferred(lambda : 75 sr.disownServiceParent()) 76 77 # WebStatus instances tend to "hang around" longer than we'd like - 78 # if there's an ongoing HTTP request, or even a connection held 79 # open by keepalive, then users may still be talking to an old 80 # WebStatus. So WebStatus objects get to keep their `master` 81 # attribute, but all other status objects lose theirs. And we want 82 # to test this without importing WebStatus, so we use name 83 if not sr.__class__.__name__.endswith('WebStatus'): 84 sr.master = None 85 86 for sr in new_config.status: 87 sr.master = self.master 88 sr.setServiceParent(self) 89 90 # reconfig any newly-added change sources, as well as existing 91 yield config.ReconfigurableServiceMixin.reconfigService(self, 92 new_config)

93

94 - def stopService(self):

95 if self._buildset_completion_sub: 96 self._buildset_completion_sub.unsubscribe() 97 self._buildset_completion_sub = None 98 if self._buildset_sub: 99 self._buildset_sub.unsubscribe() 100 self._buildset_sub = None 101 if self._build_request_sub: 102 self._build_request_sub.unsubscribe() 103 self._build_request_sub = None 104 if self._change_sub: 105 self._change_sub.unsubscribe() 106 self._change_sub = None 107 108 return service.MultiService.stopService(self)

109 110 # clean shutdown 111 112 @property

113 - def shuttingDown(self):

114 return self.botmaster.shuttingDown

115

116 - def cleanShutdown(self):

117 return self.botmaster.cleanShutdown()

118

119 - def cancelCleanShutdown(self):

120 return self.botmaster.cancelCleanShutdown()

121 122 # methods called by our clients 123

124 - def getTitle(self):

125 return self.master.config.title

126 - def getTitleURL(self):

127 return self.master.config.titleURL

128 - def getBuildbotURL(self):

129 return self.master.config.buildbotURL

130

131 - def getStatus(self):

132 # some listeners expect their .parent to be a BuildMaster object, and 133 # use this method to get the Status object. This is documented, so for 134 # now keep it working. 135 return self

136

137 - def getMetrics(self):

138 return self.master.metrics

139

140 - def getURLForBuild(self, builder_name, build_number):

141 prefix = self.getBuildbotURL() 142 return prefix + "builders/%s/builds/%d" % ( 143 urllib.quote(builder_name, safe=''), 144 build_number)

145

146 - def getURLForThing(self, thing):

147 prefix = self.getBuildbotURL() 148 if not prefix: 149 return None 150 if interfaces.IStatus.providedBy(thing): 151 return prefix 152 if interfaces.ISchedulerStatus.providedBy(thing): 153 pass 154 if interfaces.IBuilderStatus.providedBy(thing): 155 bldr = thing 156 return prefix + "builders/%s" % ( 157 urllib.quote(bldr.getName(), safe=''), 158 ) 159 if interfaces.IBuildStatus.providedBy(thing): 160 build = thing 161 bldr = build.getBuilder() 162 return self.getURLForBuild(bldr.getName(), build.getNumber()) 163 164 if interfaces.IBuildStepStatus.providedBy(thing): 165 step = thing 166 build = step.getBuild() 167 bldr = build.getBuilder() 168 return prefix + "builders/%s/builds/%d/steps/%s" % ( 169 urllib.quote(bldr.getName(), safe=''), 170 build.getNumber(), 171 urllib.quote(step.getName(), safe='')) 172 # IBuildSetStatus 173 # IBuildRequestStatus 174 # ISlaveStatus 175 if interfaces.ISlaveStatus.providedBy(thing): 176 slave = thing 177 return prefix + "buildslaves/%s" % ( 178 urllib.quote(slave.getName(), safe=''), 179 ) 180 181 # IStatusEvent 182 if interfaces.IStatusEvent.providedBy(thing): 183 # TODO: this is goofy, create IChange or something 184 if isinstance(thing, changes.Change): 185 change = thing 186 return "%schanges/%d" % (prefix, change.number) 187 188 if interfaces.IStatusLog.providedBy(thing): 189 loog = thing 190 step = loog.getStep() 191 build = step.getBuild() 192 bldr = build.getBuilder() 193 194 logs = step.getLogs() 195 for i in range(len(logs)): 196 if loog is logs[i]: 197 break 198 else: 199 return None 200 return prefix + "builders/%s/builds/%d/steps/%s/logs/%s" % ( 201 urllib.quote(bldr.getName(), safe=''), 202 build.getNumber(), 203 urllib.quote(step.getName(), safe=''), 204 urllib.quote(loog.getName(), safe=''))

205

206 - def getChangeSources(self):

207 return list(self.master.change_svc)

208

209 - def getChange(self, number):

210 """Get a Change object; returns a deferred""" 211 d = self.master.db.changes.getChange(number) 212 def chdict2change(chdict): 213 if not chdict: 214 return None 215 return changes.Change.fromChdict(self.master, chdict)

216 d.addCallback(chdict2change) 217 return d

218

219 - def getSchedulers(self):

220 return self.master.allSchedulers()

221

222 - def getBuilderNames(self, categories=None):

223 if categories == None: 224 return util.naturalSort(self.botmaster.builderNames) # don't let them break it 225 226 l = [] 227 # respect addition order 228 for name in self.botmaster.builderNames: 229 bldr = self.botmaster.builders[name] 230 if bldr.config.category in categories: 231 l.append(name) 232 return util.naturalSort(l)

233

234 - def getBuilder(self, name):

235 """ 236 @rtype: L{BuilderStatus} 237 """ 238 return self.botmaster.builders[name].builder_status

239

240 - def getSlaveNames(self):

241 return self.botmaster.slaves.keys()

242

243 - def getSlave(self, slavename):

244 return self.botmaster.slaves[slavename].slave_status

245

246 - def getBuildSets(self):

247 d = self.master.db.buildsets.getBuildsets(complete=False) 248 def make_status_objects(bsdicts): 249 return [ buildset.BuildSetStatus(bsdict, self) 250 for bsdict in bsdicts ]

251 d.addCallback(make_status_objects) 252 return d 253

254 - def generateFinishedBuilds(self, builders=[], branches=[], 255 num_builds=None, finished_before=None, 256 max_search=200):

257 258 def want_builder(bn): 259 if builders: 260 return bn in builders 261 return True

262 builder_names = [bn 263 for bn in self.getBuilderNames() 264 if want_builder(bn)] 265 266 # 'sources' is a list of generators, one for each Builder we're 267 # using. When the generator is exhausted, it is replaced in this list 268 # with None. 269 sources = [] 270 for bn in builder_names: 271 b = self.getBuilder(bn) 272 g = b.generateFinishedBuilds(branches, 273 finished_before=finished_before, 274 max_search=max_search) 275 sources.append(g) 276 277 # next_build the next build from each source 278 next_build = [None] * len(sources) 279 280 def refill(): 281 for i,g in enumerate(sources): 282 if next_build[i]: 283 # already filled 284 continue 285 if not g: 286 # already exhausted 287 continue 288 try: 289 next_build[i] = g.next() 290 except StopIteration: 291 next_build[i] = None 292 sources[i] = None 293 294 got = 0 295 while True: 296 refill() 297 # find the latest build among all the candidates 298 candidates = [(i, b, b.getTimes()[1]) 299 for i,b in enumerate(next_build) 300 if b is not None] 301 candidates.sort(lambda x,y: cmp(x[2], y[2])) 302 if not candidates: 303 return 304 305 # and remove it from the list 306 i, build, finshed_time = candidates[-1] 307 next_build[i] = None 308 got += 1 309 yield build 310 if num_builds is not None: 311 if got >= num_builds: 312 return 313

318 - def unsubscribe(self, target):

319 self.watchers.remove(target)

320 321 322 # methods called by upstream objects 323

324 - def announceNewBuilder(self, target, name, builder_status):

325 t = target.builderAdded(name, builder_status) 326 if t: 327 builder_status.subscribe(t)

328

329 - def builderAdded(self, name, basedir, category=None, description=None):

330 """ 331 @rtype: L{BuilderStatus} 332 """ 333 filename = os.path.join(self.basedir, basedir, "builder") 334 log.msg("trying to load status pickle from %s" % filename) 335 builder_status = None 336 try: 337 with open(filename, "rb") as f: 338 builder_status = load(f) 339 builder_status.master = self.master 340 341 # (bug #1068) if we need to upgrade, we probably need to rewrite 342 # this pickle, too. We determine this by looking at the list of 343 # Versioned objects that have been unpickled, and (after doUpgrade) 344 # checking to see if any of them set wasUpgraded. The Versioneds' 345 # upgradeToVersionNN methods all set this. 346 versioneds = styles.versionedsToUpgrade 347 styles.doUpgrade() 348 if True in [ hasattr(o, 'wasUpgraded') for o in versioneds.values() ]: 349 log.msg("re-writing upgraded builder pickle") 350 builder_status.saveYourself() 351 352 except IOError: 353 log.msg("no saved status pickle, creating a new one") 354 except: 355 log.msg("error while loading status pickle, creating a new one") 356 log.msg("error follows:") 357 log.err() 358 if not builder_status: 359 builder_status = builder.BuilderStatus(name, category, self.master, 360 description) 361 builder_status.addPointEvent(["builder", "created"]) 362 log.msg("added builder %s in category %s" % (name, category)) 363 # an unpickled object might not have category set from before, 364 # so set it here to make sure 365 builder_status.category = category 366 builder_status.description = description 367 builder_status.master = self.master 368 builder_status.basedir = os.path.join(self.basedir, basedir) 369 builder_status.name = name # it might have been updated 370 builder_status.status = self 371 372 if not os.path.isdir(builder_status.basedir): 373 os.makedirs(builder_status.basedir) 374 builder_status.determineNextBuildNumber() 375 376 builder_status.setBigState("offline") 377 378 for t in self.watchers: 379 self.announceNewBuilder(t, name, builder_status) 380 381 return builder_status

382

383 - def builderRemoved(self, name):

384 for t in self.watchers: 385 if hasattr(t, 'builderRemoved'): 386 t.builderRemoved(name)

387

388 - def slaveConnected(self, name):

389 for t in self.watchers: 390 if hasattr(t, 'slaveConnected'): 391 t.slaveConnected(name)

392

393 - def slaveDisconnected(self, name):

394 for t in self.watchers: 395 if hasattr(t, 'slaveDisconnected'): 396 t.slaveDisconnected(name)

397

398 - def changeAdded(self, change):

399 for t in self.watchers: 400 if hasattr(t, 'changeAdded'): 401 t.changeAdded(change)

402

403 - def asDict(self):

404 result = {} 405 # Constant 406 result['title'] = self.getTitle() 407 result['titleURL'] = self.getTitleURL() 408 result['buildbotURL'] = self.getBuildbotURL() 409 # TODO: self.getSchedulers() 410 # self.getChangeSources() 411 return result

412

413 - def build_started(self, brid, buildername, build_status):

414 if brid in self._buildreq_observers: 415 for o in self._buildreq_observers[brid]: 416 eventually(o, build_status)

417

418 - def _buildrequest_subscribe(self, brid, observer):

419 self._buildreq_observers.add(brid, observer)

420

421 - def _buildrequest_unsubscribe(self, brid, observer):

422 self._buildreq_observers.discard(brid, observer)

423

424 - def _buildset_waitUntilFinished(self, bsid):

425 d = defer.Deferred() 426 self._buildset_finished_waiters.add(bsid, d) 427 self._maybeBuildsetFinished(bsid) 428 return d

429

430 - def _maybeBuildsetFinished(self, bsid):

431 # check bsid to see if it's successful or finished, and notify anyone 432 # who cares 433 if bsid not in self._buildset_finished_waiters: 434 return 435 d = self.master.db.buildsets.getBuildset(bsid) 436 def do_notifies(bsdict): 437 bss = buildset.BuildSetStatus(bsdict, self) 438 if bss.isFinished(): 439 for d in self._buildset_finished_waiters.pop(bsid): 440 eventually(d.callback, bss)

441 d.addCallback(do_notifies) 442 d.addErrback(log.err, 'while notifying for buildset finishes') 443

444 - def _builder_subscribe(self, buildername, watcher):

445 # should get requestSubmitted and requestCancelled 446 self._builder_observers.add(buildername, watcher)

447

448 - def _builder_unsubscribe(self, buildername, watcher):

449 self._builder_observers.discard(buildername, watcher)

450

451 - def _buildsetCallback(self, **kwargs):

452 bsid = kwargs['bsid'] 453 d = self.master.db.buildsets.getBuildset(bsid) 454 def do_notifies(bsdict): 455 bss = buildset.BuildSetStatus(bsdict, self) 456 for t in self.watchers: 457 if hasattr(t, 'buildsetSubmitted'): 458 t.buildsetSubmitted(bss)

459 d.addCallback(do_notifies) 460 d.addErrback(log.err, 'while notifying buildsetSubmitted') 461

462 - def _buildsetCompletionCallback(self, bsid, result):

463 self._maybeBuildsetFinished(bsid)

464

465 - def _buildRequestCallback(self, notif):

466 buildername = notif['buildername'] 467 if buildername in self._builder_observers: 468 brs = buildrequest.BuildRequestStatus(buildername, 469 notif['brid'], self) 470 for observer in self._builder_observers[buildername]: 471 if hasattr(observer, 'requestSubmitted'): 472 eventually(observer.requestSubmitted, brs)

473

Source Code for Module buildbot.status.master