buildbot.status.master

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 import os, urllib 17 from cPickle import load 18 from twisted.python import log 19 from twisted.persisted import styles 20 from twisted.internet import defer 21 from twisted.application import service 22 from zope.interface import implements 23 from buildbot import config, interfaces, util 24 from buildbot.util import bbcollections 25 from buildbot.util.eventual import eventually 26 from buildbot.changes import changes 27 from buildbot.status import buildset, builder, buildrequest

28 29 -class Status(config.ReconfigurableServiceMixin, service.MultiService):

30 implements(interfaces.IStatus) 31

32 - def __init__(self, master):

33 service.MultiService.__init__(self) 34 self.master = master 35 self.botmaster = master.botmaster 36 self.basedir = master.basedir 37 self.watchers = [] 38 # No default limit to the log size 39 self.logMaxSize = None 40 41 self._builder_observers = bbcollections.KeyedSets() 42 self._buildreq_observers = bbcollections.KeyedSets() 43 self._buildset_finished_waiters = bbcollections.KeyedSets()

44 45 # service management 46

47 - def startService(self):

48 # subscribe to the things we need to know about 49 self._buildset_completion_sub = \ 50 self.master.subscribeToBuildsetCompletions( 51 self._buildsetCompletionCallback) 52 self._buildset_sub = \ 53 self.master.subscribeToBuildsets( 54 self._buildsetCallback) 55 self._build_request_sub = \ 56 self.master.subscribeToBuildRequests( 57 self._buildRequestCallback) 58 self._change_sub = \ 59 self.master.subscribeToChanges( 60 self.changeAdded) 61 62 return service.MultiService.startService(self)

63 64 @defer.deferredGenerator

65 - def reconfigService(self, new_config):

66 # remove the old listeners, then add the new 67 for sr in list(self): 68 wfd = defer.waitForDeferred( 69 defer.maybeDeferred(lambda : 70 sr.disownServiceParent())) 71 yield wfd 72 wfd.getResult() 73 sr.master = None 74 75 for sr in new_config.status: 76 sr.master = self.master 77 sr.setServiceParent(self) 78 79 # reconfig any newly-added change sources, as well as existing 80 wfd = defer.waitForDeferred( 81 config.ReconfigurableServiceMixin.reconfigService(self, 82 new_config)) 83 yield wfd 84 wfd.getResult()

85

86 - def stopService(self):

87 self._buildset_completion_sub.unsubscribe() 88 self._buildset_sub.unsubscribe() 89 self._build_request_sub.unsubscribe() 90 self._change_sub.unsubscribe() 91 92 return service.MultiService.stopService(self)

93 94 # clean shutdown 95 96 @property

97 - def shuttingDown(self):

98 return self.botmaster.shuttingDown

99

100 - def cleanShutdown(self):

101 return self.botmaster.cleanShutdown()

102

103 - def cancelCleanShutdown(self):

104 return self.botmaster.cancelCleanShutdown()

105 106 # methods called by our clients 107

108 - def getTitle(self):

109 return self.master.config.title

110 - def getTitleURL(self):

111 return self.master.config.titleURL

112 - def getBuildbotURL(self):

113 return self.master.config.buildbotURL

114

115 - def getStatus(self):

116 # some listeners expect their .parent to be a BuildMaster object, and 117 # use this method to get the Status object. This is documented, so for 118 # now keep it working. 119 return self

120

121 - def getMetrics(self):

122 return self.master.metrics

123

124 - def getURLForBuild(self, builder_name, build_number):

125 prefix = self.getBuildbotURL() 126 return prefix + "builders/%s/builds/%d" % ( 127 urllib.quote(builder_name, safe=''), 128 build_number)

129

130 - def getURLForThing(self, thing):

131 prefix = self.getBuildbotURL() 132 if not prefix: 133 return None 134 if interfaces.IStatus.providedBy(thing): 135 return prefix 136 if interfaces.ISchedulerStatus.providedBy(thing): 137 pass 138 if interfaces.IBuilderStatus.providedBy(thing): 139 bldr = thing 140 return prefix + "builders/%s" % ( 141 urllib.quote(bldr.getName(), safe=''), 142 ) 143 if interfaces.IBuildStatus.providedBy(thing): 144 build = thing 145 bldr = build.getBuilder() 146 return self.getURLForBuild(bldr.getName(), build.getNumber()) 147 148 if interfaces.IBuildStepStatus.providedBy(thing): 149 step = thing 150 build = step.getBuild() 151 bldr = build.getBuilder() 152 return prefix + "builders/%s/builds/%d/steps/%s" % ( 153 urllib.quote(bldr.getName(), safe=''), 154 build.getNumber(), 155 urllib.quote(step.getName(), safe='')) 156 # IBuildSetStatus 157 # IBuildRequestStatus 158 # ISlaveStatus 159 160 # IStatusEvent 161 if interfaces.IStatusEvent.providedBy(thing): 162 # TODO: this is goofy, create IChange or something 163 if isinstance(thing, changes.Change): 164 change = thing 165 return "%schanges/%d" % (prefix, change.number) 166 167 if interfaces.IStatusLog.providedBy(thing): 168 loog = thing 169 step = loog.getStep() 170 build = step.getBuild() 171 bldr = build.getBuilder() 172 173 logs = step.getLogs() 174 for i in range(len(logs)): 175 if loog is logs[i]: 176 break 177 else: 178 return None 179 return prefix + "builders/%s/builds/%d/steps/%s/logs/%s" % ( 180 urllib.quote(bldr.getName(), safe=''), 181 build.getNumber(), 182 urllib.quote(step.getName(), safe=''), 183 urllib.quote(loog.getName(), safe=''))

184

185 - def getChangeSources(self):

186 return list(self.master.change_svc)

187

188 - def getChange(self, number):

189 """Get a Change object; returns a deferred""" 190 d = self.master.db.changes.getChange(number) 191 def chdict2change(chdict): 192 if not chdict: 193 return None 194 return changes.Change.fromChdict(self.master, chdict)

195 d.addCallback(chdict2change) 196 return d

197

198 - def getSchedulers(self):

199 return self.master.allSchedulers()

200

201 - def getBuilderNames(self, categories=None):

202 if categories == None: 203 return util.naturalSort(self.botmaster.builderNames) # don't let them break it 204 205 l = [] 206 # respect addition order 207 for name in self.botmaster.builderNames: 208 bldr = self.botmaster.builders[name] 209 if bldr.config.category in categories: 210 l.append(name) 211 return util.naturalSort(l)

212

213 - def getBuilder(self, name):

214 """ 215 @rtype: L{BuilderStatus} 216 """ 217 return self.botmaster.builders[name].builder_status

218

219 - def getSlaveNames(self):

220 return self.botmaster.slaves.keys()

221

222 - def getSlave(self, slavename):

223 return self.botmaster.slaves[slavename].slave_status

224

225 - def getBuildSets(self):

226 d = self.master.db.buildsets.getBuildsets(complete=False) 227 def make_status_objects(bsdicts): 228 return [ buildset.BuildSetStatus(bsdict, self) 229 for bsdict in bsdicts ]

230 d.addCallback(make_status_objects) 231 return d 232

233 - def generateFinishedBuilds(self, builders=[], branches=[], 234 num_builds=None, finished_before=None, 235 max_search=200):

236 237 def want_builder(bn): 238 if builders: 239 return bn in builders 240 return True

241 builder_names = [bn 242 for bn in self.getBuilderNames() 243 if want_builder(bn)] 244 245 # 'sources' is a list of generators, one for each Builder we're 246 # using. When the generator is exhausted, it is replaced in this list 247 # with None. 248 sources = [] 249 for bn in builder_names: 250 b = self.getBuilder(bn) 251 g = b.generateFinishedBuilds(branches, 252 finished_before=finished_before, 253 max_search=max_search) 254 sources.append(g) 255 256 # next_build the next build from each source 257 next_build = [None] * len(sources) 258 259 def refill(): 260 for i,g in enumerate(sources): 261 if next_build[i]: 262 # already filled 263 continue 264 if not g: 265 # already exhausted 266 continue 267 try: 268 next_build[i] = g.next() 269 except StopIteration: 270 next_build[i] = None 271 sources[i] = None 272 273 got = 0 274 while True: 275 refill() 276 # find the latest build among all the candidates 277 candidates = [(i, b, b.getTimes()[1]) 278 for i,b in enumerate(next_build) 279 if b is not None] 280 candidates.sort(lambda x,y: cmp(x[2], y[2])) 281 if not candidates: 282 return 283 284 # and remove it from the list 285 i, build, finshed_time = candidates[-1] 286 next_build[i] = None 287 got += 1 288 yield build 289 if num_builds is not None: 290 if got >= num_builds: 291 return 292

297 - def unsubscribe(self, target):

298 self.watchers.remove(target)

299 300 301 # methods called by upstream objects 302

303 - def announceNewBuilder(self, target, name, builder_status):

304 t = target.builderAdded(name, builder_status) 305 if t: 306 builder_status.subscribe(t)

307

308 - def builderAdded(self, name, basedir, category=None):

309 """ 310 @rtype: L{BuilderStatus} 311 """ 312 filename = os.path.join(self.basedir, basedir, "builder") 313 log.msg("trying to load status pickle from %s" % filename) 314 builder_status = None 315 try: 316 builder_status = load(open(filename, "rb")) 317 builder_status.master = self.master 318 319 # (bug #1068) if we need to upgrade, we probably need to rewrite 320 # this pickle, too. We determine this by looking at the list of 321 # Versioned objects that have been unpickled, and (after doUpgrade) 322 # checking to see if any of them set wasUpgraded. The Versioneds' 323 # upgradeToVersionNN methods all set this. 324 versioneds = styles.versionedsToUpgrade 325 styles.doUpgrade() 326 if True in [ hasattr(o, 'wasUpgraded') for o in versioneds.values() ]: 327 log.msg("re-writing upgraded builder pickle") 328 builder_status.saveYourself() 329 330 except IOError: 331 log.msg("no saved status pickle, creating a new one") 332 except: 333 log.msg("error while loading status pickle, creating a new one") 334 log.msg("error follows:") 335 log.err() 336 if not builder_status: 337 builder_status = builder.BuilderStatus(name, category, self.master) 338 builder_status.addPointEvent(["builder", "created"]) 339 log.msg("added builder %s in category %s" % (name, category)) 340 # an unpickled object might not have category set from before, 341 # so set it here to make sure 342 builder_status.master = self.master 343 builder_status.basedir = os.path.join(self.basedir, basedir) 344 builder_status.name = name # it might have been updated 345 builder_status.status = self 346 347 if not os.path.isdir(builder_status.basedir): 348 os.makedirs(builder_status.basedir) 349 builder_status.determineNextBuildNumber() 350 351 builder_status.setBigState("offline") 352 353 for t in self.watchers: 354 self.announceNewBuilder(t, name, builder_status) 355 356 return builder_status

357

358 - def builderRemoved(self, name):

359 for t in self.watchers: 360 if hasattr(t, 'builderRemoved'): 361 t.builderRemoved(name)

362

363 - def slaveConnected(self, name):

364 for t in self.watchers: 365 if hasattr(t, 'slaveConnected'): 366 t.slaveConnected(name)

367

368 - def slaveDisconnected(self, name):

369 for t in self.watchers: 370 if hasattr(t, 'slaveDisconnected'): 371 t.slaveDisconnected(name)

372

373 - def changeAdded(self, change):

374 for t in self.watchers: 375 if hasattr(t, 'changeAdded'): 376 t.changeAdded(change)

377

378 - def asDict(self):

379 result = {} 380 # Constant 381 result['title'] = self.getTitle() 382 result['titleURL'] = self.getTitleURL() 383 result['buildbotURL'] = self.getBuildbotURL() 384 # TODO: self.getSchedulers() 385 # self.getChangeSources() 386 return result

387

388 - def build_started(self, brid, buildername, build_status):

389 if brid in self._buildreq_observers: 390 for o in self._buildreq_observers[brid]: 391 eventually(o, build_status)

392

393 - def _buildrequest_subscribe(self, brid, observer):

394 self._buildreq_observers.add(brid, observer)

395

396 - def _buildrequest_unsubscribe(self, brid, observer):

397 self._buildreq_observers.discard(brid, observer)

398

399 - def _buildset_waitUntilFinished(self, bsid):

400 d = defer.Deferred() 401 self._buildset_finished_waiters.add(bsid, d) 402 self._maybeBuildsetFinished(bsid) 403 return d

404

405 - def _maybeBuildsetFinished(self, bsid):

406 # check bsid to see if it's successful or finished, and notify anyone 407 # who cares 408 if bsid not in self._buildset_finished_waiters: 409 return 410 d = self.master.db.buildsets.getBuildset(bsid) 411 def do_notifies(bsdict): 412 bss = buildset.BuildSetStatus(bsdict, self) 413 if bss.isFinished(): 414 for d in self._buildset_finished_waiters.pop(bsid): 415 eventually(d.callback, bss)

416 d.addCallback(do_notifies) 417 d.addErrback(log.err, 'while notifying for buildset finishes') 418

419 - def _builder_subscribe(self, buildername, watcher):

420 # should get requestSubmitted and requestCancelled 421 self._builder_observers.add(buildername, watcher)

422

423 - def _builder_unsubscribe(self, buildername, watcher):

424 self._builder_observers.discard(buildername, watcher)

425

426 - def _buildsetCallback(self, **kwargs):

427 bsid = kwargs['bsid'] 428 d = self.master.db.buildsets.getBuildset(bsid) 429 def do_notifies(bsdict): 430 bss = buildset.BuildSetStatus(bsdict, self) 431 for t in self.watchers: 432 if hasattr(t, 'buildsetSubmitted'): 433 t.buildsetSubmitted(bss)

434 d.addCallback(do_notifies) 435 d.addErrback(log.err, 'while notifying buildsetSubmitted') 436

437 - def _buildsetCompletionCallback(self, bsid, result):

438 self._maybeBuildsetFinished(bsid)

439

440 - def _buildRequestCallback(self, notif):

441 buildername = notif['buildername'] 442 if buildername in self._builder_observers: 443 brs = buildrequest.BuildRequestStatus(buildername, 444 notif['brid'], self) 445 for observer in self._builder_observers[buildername]: 446 if hasattr(observer, 'requestSubmitted'): 447 eventually(observer.requestSubmitted, brs)

448

Source Code for Module buildbot.status.master