buildbot.status.master

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 from __future__ import with_statement 17 18 import os, urllib 19 from cPickle import load 20 from twisted.python import log 21 from twisted.persisted import styles 22 from twisted.internet import defer 23 from twisted.application import service 24 from zope.interface import implements 25 from buildbot import config, interfaces, util 26 from buildbot.util import bbcollections 27 from buildbot.util.eventual import eventually 28 from buildbot.changes import changes 29 from buildbot.status import buildset, builder, buildrequest

30 31 -class Status(config.ReconfigurableServiceMixin, service.MultiService):

32 implements(interfaces.IStatus) 33

34 - def __init__(self, master):

35 service.MultiService.__init__(self) 36 self.master = master 37 self.botmaster = master.botmaster 38 self.basedir = master.basedir 39 self.watchers = [] 40 # No default limit to the log size 41 self.logMaxSize = None 42 43 self._builder_observers = bbcollections.KeyedSets() 44 self._buildreq_observers = bbcollections.KeyedSets() 45 self._buildset_finished_waiters = bbcollections.KeyedSets()

46 47 # service management 48

49 - def startService(self):

50 # subscribe to the things we need to know about 51 self._buildset_completion_sub = \ 52 self.master.subscribeToBuildsetCompletions( 53 self._buildsetCompletionCallback) 54 self._buildset_sub = \ 55 self.master.subscribeToBuildsets( 56 self._buildsetCallback) 57 self._build_request_sub = \ 58 self.master.subscribeToBuildRequests( 59 self._buildRequestCallback) 60 self._change_sub = \ 61 self.master.subscribeToChanges( 62 self.changeAdded) 63 64 return service.MultiService.startService(self)

65 66 @defer.inlineCallbacks

67 - def reconfigService(self, new_config):

68 # remove the old listeners, then add the new 69 for sr in list(self): 70 yield defer.maybeDeferred(lambda : 71 sr.disownServiceParent()) 72 73 # WebStatus instances tend to "hang around" longer than we'd like - 74 # if there's an ongoing HTTP request, or even a connection held 75 # open by keepalive, then users may still be talking to an old 76 # WebStatus. So WebStatus objects get to keep their `master` 77 # attribute, but all other status objects lose theirs. And we want 78 # to test this without importing WebStatus, so we use name 79 if not sr.__class__.__name__.endswith('WebStatus'): 80 sr.master = None 81 82 for sr in new_config.status: 83 sr.master = self.master 84 sr.setServiceParent(self) 85 86 # reconfig any newly-added change sources, as well as existing 87 yield config.ReconfigurableServiceMixin.reconfigService(self, 88 new_config)

89

90 - def stopService(self):

91 self._buildset_completion_sub.unsubscribe() 92 self._buildset_sub.unsubscribe() 93 self._build_request_sub.unsubscribe() 94 self._change_sub.unsubscribe() 95 96 return service.MultiService.stopService(self)

97 98 # clean shutdown 99 100 @property

101 - def shuttingDown(self):

102 return self.botmaster.shuttingDown

103

104 - def cleanShutdown(self):

105 return self.botmaster.cleanShutdown()

106

107 - def cancelCleanShutdown(self):

108 return self.botmaster.cancelCleanShutdown()

109 110 # methods called by our clients 111

112 - def getTitle(self):

113 return self.master.config.title

114 - def getTitleURL(self):

115 return self.master.config.titleURL

116 - def getBuildbotURL(self):

117 return self.master.config.buildbotURL

118

119 - def getStatus(self):

120 # some listeners expect their .parent to be a BuildMaster object, and 121 # use this method to get the Status object. This is documented, so for 122 # now keep it working. 123 return self

124

125 - def getMetrics(self):

126 return self.master.metrics

127

128 - def getURLForBuild(self, builder_name, build_number):

129 prefix = self.getBuildbotURL() 130 return prefix + "builders/%s/builds/%d" % ( 131 urllib.quote(builder_name, safe=''), 132 build_number)

133

134 - def getURLForThing(self, thing):

135 prefix = self.getBuildbotURL() 136 if not prefix: 137 return None 138 if interfaces.IStatus.providedBy(thing): 139 return prefix 140 if interfaces.ISchedulerStatus.providedBy(thing): 141 pass 142 if interfaces.IBuilderStatus.providedBy(thing): 143 bldr = thing 144 return prefix + "builders/%s" % ( 145 urllib.quote(bldr.getName(), safe=''), 146 ) 147 if interfaces.IBuildStatus.providedBy(thing): 148 build = thing 149 bldr = build.getBuilder() 150 return self.getURLForBuild(bldr.getName(), build.getNumber()) 151 152 if interfaces.IBuildStepStatus.providedBy(thing): 153 step = thing 154 build = step.getBuild() 155 bldr = build.getBuilder() 156 return prefix + "builders/%s/builds/%d/steps/%s" % ( 157 urllib.quote(bldr.getName(), safe=''), 158 build.getNumber(), 159 urllib.quote(step.getName(), safe='')) 160 # IBuildSetStatus 161 # IBuildRequestStatus 162 # ISlaveStatus 163 164 # IStatusEvent 165 if interfaces.IStatusEvent.providedBy(thing): 166 # TODO: this is goofy, create IChange or something 167 if isinstance(thing, changes.Change): 168 change = thing 169 return "%schanges/%d" % (prefix, change.number) 170 171 if interfaces.IStatusLog.providedBy(thing): 172 loog = thing 173 step = loog.getStep() 174 build = step.getBuild() 175 bldr = build.getBuilder() 176 177 logs = step.getLogs() 178 for i in range(len(logs)): 179 if loog is logs[i]: 180 break 181 else: 182 return None 183 return prefix + "builders/%s/builds/%d/steps/%s/logs/%s" % ( 184 urllib.quote(bldr.getName(), safe=''), 185 build.getNumber(), 186 urllib.quote(step.getName(), safe=''), 187 urllib.quote(loog.getName(), safe=''))

188

189 - def getChangeSources(self):

190 return list(self.master.change_svc)

191

192 - def getChange(self, number):

193 """Get a Change object; returns a deferred""" 194 d = self.master.db.changes.getChange(number) 195 def chdict2change(chdict): 196 if not chdict: 197 return None 198 return changes.Change.fromChdict(self.master, chdict)

199 d.addCallback(chdict2change) 200 return d

201

202 - def getSchedulers(self):

203 return self.master.allSchedulers()

204

205 - def getBuilderNames(self, categories=None):

206 if categories == None: 207 return util.naturalSort(self.botmaster.builderNames) # don't let them break it 208 209 l = [] 210 # respect addition order 211 for name in self.botmaster.builderNames: 212 bldr = self.botmaster.builders[name] 213 if bldr.config.category in categories: 214 l.append(name) 215 return util.naturalSort(l)

216

217 - def getBuilder(self, name):

218 """ 219 @rtype: L{BuilderStatus} 220 """ 221 return self.botmaster.builders[name].builder_status

222

223 - def getSlaveNames(self):

224 return self.botmaster.slaves.keys()

225

226 - def getSlave(self, slavename):

227 return self.botmaster.slaves[slavename].slave_status

228

229 - def getBuildSets(self):

230 d = self.master.db.buildsets.getBuildsets(complete=False) 231 def make_status_objects(bsdicts): 232 return [ buildset.BuildSetStatus(bsdict, self) 233 for bsdict in bsdicts ]

234 d.addCallback(make_status_objects) 235 return d 236

237 - def generateFinishedBuilds(self, builders=[], branches=[], 238 num_builds=None, finished_before=None, 239 max_search=200):

240 241 def want_builder(bn): 242 if builders: 243 return bn in builders 244 return True

245 builder_names = [bn 246 for bn in self.getBuilderNames() 247 if want_builder(bn)] 248 249 # 'sources' is a list of generators, one for each Builder we're 250 # using. When the generator is exhausted, it is replaced in this list 251 # with None. 252 sources = [] 253 for bn in builder_names: 254 b = self.getBuilder(bn) 255 g = b.generateFinishedBuilds(branches, 256 finished_before=finished_before, 257 max_search=max_search) 258 sources.append(g) 259 260 # next_build the next build from each source 261 next_build = [None] * len(sources) 262 263 def refill(): 264 for i,g in enumerate(sources): 265 if next_build[i]: 266 # already filled 267 continue 268 if not g: 269 # already exhausted 270 continue 271 try: 272 next_build[i] = g.next() 273 except StopIteration: 274 next_build[i] = None 275 sources[i] = None 276 277 got = 0 278 while True: 279 refill() 280 # find the latest build among all the candidates 281 candidates = [(i, b, b.getTimes()[1]) 282 for i,b in enumerate(next_build) 283 if b is not None] 284 candidates.sort(lambda x,y: cmp(x[2], y[2])) 285 if not candidates: 286 return 287 288 # and remove it from the list 289 i, build, finshed_time = candidates[-1] 290 next_build[i] = None 291 got += 1 292 yield build 293 if num_builds is not None: 294 if got >= num_builds: 295 return 296

301 - def unsubscribe(self, target):

302 self.watchers.remove(target)

303 304 305 # methods called by upstream objects 306

307 - def announceNewBuilder(self, target, name, builder_status):

308 t = target.builderAdded(name, builder_status) 309 if t: 310 builder_status.subscribe(t)

311

312 - def builderAdded(self, name, basedir, category=None):

313 """ 314 @rtype: L{BuilderStatus} 315 """ 316 filename = os.path.join(self.basedir, basedir, "builder") 317 log.msg("trying to load status pickle from %s" % filename) 318 builder_status = None 319 try: 320 with open(filename, "rb") as f: 321 builder_status = load(f) 322 builder_status.master = self.master 323 324 # (bug #1068) if we need to upgrade, we probably need to rewrite 325 # this pickle, too. We determine this by looking at the list of 326 # Versioned objects that have been unpickled, and (after doUpgrade) 327 # checking to see if any of them set wasUpgraded. The Versioneds' 328 # upgradeToVersionNN methods all set this. 329 versioneds = styles.versionedsToUpgrade 330 styles.doUpgrade() 331 if True in [ hasattr(o, 'wasUpgraded') for o in versioneds.values() ]: 332 log.msg("re-writing upgraded builder pickle") 333 builder_status.saveYourself() 334 335 except IOError: 336 log.msg("no saved status pickle, creating a new one") 337 except: 338 log.msg("error while loading status pickle, creating a new one") 339 log.msg("error follows:") 340 log.err() 341 if not builder_status: 342 builder_status = builder.BuilderStatus(name, category, self.master) 343 builder_status.addPointEvent(["builder", "created"]) 344 log.msg("added builder %s in category %s" % (name, category)) 345 # an unpickled object might not have category set from before, 346 # so set it here to make sure 347 builder_status.category = category 348 builder_status.master = self.master 349 builder_status.basedir = os.path.join(self.basedir, basedir) 350 builder_status.name = name # it might have been updated 351 builder_status.status = self 352 353 if not os.path.isdir(builder_status.basedir): 354 os.makedirs(builder_status.basedir) 355 builder_status.determineNextBuildNumber() 356 357 builder_status.setBigState("offline") 358 359 for t in self.watchers: 360 self.announceNewBuilder(t, name, builder_status) 361 362 return builder_status

363

364 - def builderRemoved(self, name):

365 for t in self.watchers: 366 if hasattr(t, 'builderRemoved'): 367 t.builderRemoved(name)

368

369 - def slaveConnected(self, name):

370 for t in self.watchers: 371 if hasattr(t, 'slaveConnected'): 372 t.slaveConnected(name)

373

374 - def slaveDisconnected(self, name):

375 for t in self.watchers: 376 if hasattr(t, 'slaveDisconnected'): 377 t.slaveDisconnected(name)

378

379 - def changeAdded(self, change):

380 for t in self.watchers: 381 if hasattr(t, 'changeAdded'): 382 t.changeAdded(change)

383

384 - def asDict(self):

385 result = {} 386 # Constant 387 result['title'] = self.getTitle() 388 result['titleURL'] = self.getTitleURL() 389 result['buildbotURL'] = self.getBuildbotURL() 390 # TODO: self.getSchedulers() 391 # self.getChangeSources() 392 return result

393

394 - def build_started(self, brid, buildername, build_status):

395 if brid in self._buildreq_observers: 396 for o in self._buildreq_observers[brid]: 397 eventually(o, build_status)

398

399 - def _buildrequest_subscribe(self, brid, observer):

400 self._buildreq_observers.add(brid, observer)

401

402 - def _buildrequest_unsubscribe(self, brid, observer):

403 self._buildreq_observers.discard(brid, observer)

404

405 - def _buildset_waitUntilFinished(self, bsid):

406 d = defer.Deferred() 407 self._buildset_finished_waiters.add(bsid, d) 408 self._maybeBuildsetFinished(bsid) 409 return d

410

411 - def _maybeBuildsetFinished(self, bsid):

412 # check bsid to see if it's successful or finished, and notify anyone 413 # who cares 414 if bsid not in self._buildset_finished_waiters: 415 return 416 d = self.master.db.buildsets.getBuildset(bsid) 417 def do_notifies(bsdict): 418 bss = buildset.BuildSetStatus(bsdict, self) 419 if bss.isFinished(): 420 for d in self._buildset_finished_waiters.pop(bsid): 421 eventually(d.callback, bss)

422 d.addCallback(do_notifies) 423 d.addErrback(log.err, 'while notifying for buildset finishes') 424

425 - def _builder_subscribe(self, buildername, watcher):

426 # should get requestSubmitted and requestCancelled 427 self._builder_observers.add(buildername, watcher)

428

429 - def _builder_unsubscribe(self, buildername, watcher):

430 self._builder_observers.discard(buildername, watcher)

431

432 - def _buildsetCallback(self, **kwargs):

433 bsid = kwargs['bsid'] 434 d = self.master.db.buildsets.getBuildset(bsid) 435 def do_notifies(bsdict): 436 bss = buildset.BuildSetStatus(bsdict, self) 437 for t in self.watchers: 438 if hasattr(t, 'buildsetSubmitted'): 439 t.buildsetSubmitted(bss)

440 d.addCallback(do_notifies) 441 d.addErrback(log.err, 'while notifying buildsetSubmitted') 442

443 - def _buildsetCompletionCallback(self, bsid, result):

444 self._maybeBuildsetFinished(bsid)

445

446 - def _buildRequestCallback(self, notif):

447 buildername = notif['buildername'] 448 if buildername in self._builder_observers: 449 brs = buildrequest.BuildRequestStatus(buildername, 450 notif['brid'], self) 451 for observer in self._builder_observers[buildername]: 452 if hasattr(observer, 'requestSubmitted'): 453 eventually(observer.requestSubmitted, brs)

454

Source Code for Module buildbot.status.master