buildbot.status.master

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 import os, urllib 17 from cPickle import load 18 from twisted.python import log 19 from twisted.persisted import styles 20 from twisted.internet import defer 21 from zope.interface import implements 22 from buildbot import interfaces 23 from buildbot.util import bbcollections 24 from buildbot.util.eventual import eventually 25 from buildbot.changes import changes 26 from buildbot.status import buildset, builder, buildrequest

27 28 -class Status:

29 """ 30 I represent the status of the buildmaster. 31 """ 32 implements(interfaces.IStatus) 33

34 - def __init__(self, master):

35 self.master = master 36 self.botmaster = master.botmaster 37 self.db = None 38 self.basedir = master.basedir 39 self.watchers = [] 40 # compress logs bigger than 4k, a good default on linux 41 self.logCompressionLimit = 4*1024 42 self.logCompressionMethod = "bz2" 43 # No default limit to the log size 44 self.logMaxSize = None 45 self.logMaxTailSize = None 46 47 # subscribe to the things we need to know about 48 self.master.subscribeToBuildsetCompletions( 49 self._buildsetCompletionCallback) 50 self.master.subscribeToBuildsets( 51 self._buildsetCallback) 52 self.master.subscribeToBuildRequests( 53 self._buildRequestCallback) 54 55 self._builder_observers = bbcollections.KeyedSets() 56 self._buildreq_observers = bbcollections.KeyedSets() 57 self._buildset_finished_waiters = bbcollections.KeyedSets()

58 59 @property

60 - def shuttingDown(self):

61 return self.botmaster.shuttingDown

62

63 - def cleanShutdown(self):

64 return self.botmaster.cleanShutdown()

65

66 - def cancelCleanShutdown(self):

67 return self.botmaster.cancelCleanShutdown()

68 69 # methods called by our clients 70

71 - def getTitle(self):

72 return self.master.title

73 - def getTitleURL(self):

74 return self.master.titleURL

75 - def getBuildbotURL(self):

76 return self.master.buildbotURL

77

78 - def getMetrics(self):

79 return self.master.metrics

80

81 - def getURLForThing(self, thing):

82 prefix = self.getBuildbotURL() 83 if not prefix: 84 return None 85 if interfaces.IStatus.providedBy(thing): 86 return prefix 87 if interfaces.ISchedulerStatus.providedBy(thing): 88 pass 89 if interfaces.IBuilderStatus.providedBy(thing): 90 bldr = thing 91 return prefix + "builders/%s" % ( 92 urllib.quote(bldr.getName(), safe=''), 93 ) 94 if interfaces.IBuildStatus.providedBy(thing): 95 build = thing 96 bldr = build.getBuilder() 97 return prefix + "builders/%s/builds/%d" % ( 98 urllib.quote(bldr.getName(), safe=''), 99 build.getNumber()) 100 if interfaces.IBuildStepStatus.providedBy(thing): 101 step = thing 102 build = step.getBuild() 103 bldr = build.getBuilder() 104 return prefix + "builders/%s/builds/%d/steps/%s" % ( 105 urllib.quote(bldr.getName(), safe=''), 106 build.getNumber(), 107 urllib.quote(step.getName(), safe='')) 108 # IBuildSetStatus 109 # IBuildRequestStatus 110 # ISlaveStatus 111 112 # IStatusEvent 113 if interfaces.IStatusEvent.providedBy(thing): 114 # TODO: this is goofy, create IChange or something 115 if isinstance(thing, changes.Change): 116 change = thing 117 return "%schanges/%d" % (prefix, change.number) 118 119 if interfaces.IStatusLog.providedBy(thing): 120 loog = thing 121 step = loog.getStep() 122 build = step.getBuild() 123 bldr = build.getBuilder() 124 125 logs = step.getLogs() 126 for i in range(len(logs)): 127 if loog is logs[i]: 128 break 129 else: 130 return None 131 return prefix + "builders/%s/builds/%d/steps/%s/logs/%s" % ( 132 urllib.quote(bldr.getName(), safe=''), 133 build.getNumber(), 134 urllib.quote(step.getName(), safe=''), 135 urllib.quote(loog.getName()))

136

137 - def getChangeSources(self):

138 return list(self.master.change_svc)

139

140 - def getChange(self, number):

141 """Get a Change object; returns a deferred""" 142 d = self.master.db.changes.getChange(number) 143 def chdict2change(chdict): 144 if not chdict: 145 return None 146 return changes.Change.fromChdict(self.master, chdict)

147 d.addCallback(chdict2change) 148 return d

149

150 - def getSchedulers(self):

151 return self.master.allSchedulers()

152

153 - def getBuilderNames(self, categories=None):

154 if categories == None: 155 return self.botmaster.builderNames[:] # don't let them break it 156 157 l = [] 158 # respect addition order 159 for name in self.botmaster.builderNames: 160 bldr = self.botmaster.builders[name] 161 if bldr.builder_status.category in categories: 162 l.append(name) 163 return l

164

165 - def getBuilder(self, name):

166 """ 167 @rtype: L{BuilderStatus} 168 """ 169 return self.botmaster.builders[name].builder_status

170

171 - def getSlaveNames(self):

172 return self.botmaster.slaves.keys()

173

174 - def getSlave(self, slavename):

175 return self.botmaster.slaves[slavename].slave_status

176

177 - def getBuildSets(self):

178 d = self.master.db.buildsets.getBuildsets(complete=False) 179 def make_status_objects(bsdicts): 180 return [ buildset.BuildSetStatus(bsdict, self) 181 for bsdict in bsdicts ]

182 d.addCallback(make_status_objects) 183 return d 184

185 - def generateFinishedBuilds(self, builders=[], branches=[], 186 num_builds=None, finished_before=None, 187 max_search=200):

188 189 def want_builder(bn): 190 if builders: 191 return bn in builders 192 return True

193 builder_names = [bn 194 for bn in self.getBuilderNames() 195 if want_builder(bn)] 196 197 # 'sources' is a list of generators, one for each Builder we're 198 # using. When the generator is exhausted, it is replaced in this list 199 # with None. 200 sources = [] 201 for bn in builder_names: 202 b = self.getBuilder(bn) 203 g = b.generateFinishedBuilds(branches, 204 finished_before=finished_before, 205 max_search=max_search) 206 sources.append(g) 207 208 # next_build the next build from each source 209 next_build = [None] * len(sources) 210 211 def refill(): 212 for i,g in enumerate(sources): 213 if next_build[i]: 214 # already filled 215 continue 216 if not g: 217 # already exhausted 218 continue 219 try: 220 next_build[i] = g.next() 221 except StopIteration: 222 next_build[i] = None 223 sources[i] = None 224 225 got = 0 226 while True: 227 refill() 228 # find the latest build among all the candidates 229 candidates = [(i, b, b.getTimes()[1]) 230 for i,b in enumerate(next_build) 231 if b is not None] 232 candidates.sort(lambda x,y: cmp(x[2], y[2])) 233 if not candidates: 234 return 235 236 # and remove it from the list 237 i, build, finshed_time = candidates[-1] 238 next_build[i] = None 239 got += 1 240 yield build 241 if num_builds is not None: 242 if got >= num_builds: 243 return 244

249 - def unsubscribe(self, target):

250 self.watchers.remove(target)

251 252 253 # methods called by upstream objects 254

255 - def announceNewBuilder(self, target, name, builder_status):

256 t = target.builderAdded(name, builder_status) 257 if t: 258 builder_status.subscribe(t)

259

260 - def builderAdded(self, name, basedir, category=None):

261 """ 262 @rtype: L{BuilderStatus} 263 """ 264 filename = os.path.join(self.basedir, basedir, "builder") 265 log.msg("trying to load status pickle from %s" % filename) 266 builder_status = None 267 try: 268 builder_status = load(open(filename, "rb")) 269 270 # (bug #1068) if we need to upgrade, we probably need to rewrite 271 # this pickle, too. We determine this by looking at the list of 272 # Versioned objects that have been unpickled, and (after doUpgrade) 273 # checking to see if any of them set wasUpgraded. The Versioneds' 274 # upgradeToVersionNN methods all set this. 275 versioneds = styles.versionedsToUpgrade 276 styles.doUpgrade() 277 if True in [ hasattr(o, 'wasUpgraded') for o in versioneds.values() ]: 278 log.msg("re-writing upgraded builder pickle") 279 builder_status.saveYourself() 280 281 except IOError: 282 log.msg("no saved status pickle, creating a new one") 283 except: 284 log.msg("error while loading status pickle, creating a new one") 285 log.msg("error follows:") 286 log.err() 287 if not builder_status: 288 builder_status = builder.BuilderStatus(name, category) 289 builder_status.addPointEvent(["builder", "created"]) 290 log.msg("added builder %s in category %s" % (name, category)) 291 # an unpickled object might not have category set from before, 292 # so set it here to make sure 293 builder_status.category = category 294 builder_status.basedir = os.path.join(self.basedir, basedir) 295 builder_status.name = name # it might have been updated 296 builder_status.status = self 297 298 if not os.path.isdir(builder_status.basedir): 299 os.makedirs(builder_status.basedir) 300 builder_status.determineNextBuildNumber() 301 302 builder_status.setBigState("offline") 303 builder_status.setLogCompressionLimit(self.logCompressionLimit) 304 builder_status.setLogCompressionMethod(self.logCompressionMethod) 305 builder_status.setLogMaxSize(self.logMaxSize) 306 builder_status.setLogMaxTailSize(self.logMaxTailSize) 307 308 for t in self.watchers: 309 self.announceNewBuilder(t, name, builder_status) 310 311 return builder_status

312

313 - def builderRemoved(self, name):

314 for t in self.watchers: 315 if hasattr(t, 'builderRemoved'): 316 t.builderRemoved(name)

317

318 - def slaveConnected(self, name):

319 for t in self.watchers: 320 if hasattr(t, 'slaveConnected'): 321 t.slaveConnected(name)

322

323 - def slaveDisconnected(self, name):

324 for t in self.watchers: 325 if hasattr(t, 'slaveDisconnected'): 326 t.slaveDisconnected(name)

327

328 - def changeAdded(self, change):

329 for t in self.watchers: 330 if hasattr(t, 'changeAdded'): 331 t.changeAdded(change)

332

333 - def asDict(self):

334 result = {} 335 # Constant 336 result['title'] = self.getTitle() 337 result['titleURL'] = self.getTitleURL() 338 result['buildbotURL'] = self.getBuildbotURL() 339 # TODO: self.getSchedulers() 340 # self.getChangeSources() 341 return result

342

343 - def buildreqs_retired(self, requests):

344 for r in requests: 345 #r.id: notify subscribers (none right now) 346 # r.bsid: check for completion, notify subscribers, unsubscribe 347 pass

348

349 - def build_started(self, brid, buildername, build_status):

350 if brid in self._buildreq_observers: 351 for o in self._buildreq_observers[brid]: 352 eventually(o, build_status)

353

354 - def _buildrequest_subscribe(self, brid, observer):

355 self._buildreq_observers.add(brid, observer)

356

357 - def _buildrequest_unsubscribe(self, brid, observer):

358 self._buildreq_observers.discard(brid, observer)

359

360 - def _buildset_waitUntilFinished(self, bsid):

361 d = defer.Deferred() 362 self._buildset_finished_waiters.add(bsid, d) 363 self._maybeBuildsetFinished(bsid) 364 return d

365

366 - def _maybeBuildsetFinished(self, bsid):

367 # check bsid to see if it's successful or finished, and notify anyone 368 # who cares 369 if bsid not in self._buildset_finished_waiters: 370 return 371 d = self.master.db.buildsets.getBuildset(bsid) 372 def do_notifies(bsdict): 373 bss = buildset.BuildSetStatus(bsdict, self) 374 if bss.isFinished(): 375 for d in self._buildset_finished_waiters.pop(bsid): 376 eventually(d.callback, bss)

377 d.addCallback(do_notifies) 378 d.addErrback(log.err, 'while notifying for buildset finishes') 379

380 - def _builder_subscribe(self, buildername, watcher):

381 # should get requestSubmitted and requestCancelled 382 self._builder_observers.add(buildername, watcher)

383

384 - def _builder_unsubscribe(self, buildername, watcher):

385 self._builder_observers.discard(buildername, watcher)

386

387 - def _buildsetCallback(self, **kwargs):

388 bsid = kwargs['bsid'] 389 d = self.master.db.buildsets.getBuildset(bsid) 390 def do_notifies(bsdict): 391 bss = buildset.BuildSetStatus(bsdict, self) 392 for t in self.watchers: 393 if hasattr(t, 'buildsetSubmitted'): 394 t.buildsetSubmitted(bss)

395 d.addCallback(do_notifies) 396 d.addErrback(log.err, 'while notifying buildsetSubmitted') 397

398 - def _buildsetCompletionCallback(self, bsid, result):

399 self._maybeBuildsetFinished(bsid)

400

401 - def _buildRequestCallback(self, notif):

402 buildername = notif['buildername'] 403 if buildername in self._builder_observers: 404 brs = buildrequest.BuildRequestStatus(buildername, 405 notif['brid'], self) 406 for observer in self._builder_observers[buildername]: 407 if hasattr(observer, 'requestSubmitted'): 408 eventually(observer.requestSubmitted, brs)

409

Source Code for Module buildbot.status.master