Package buildbot :: Package status :: Module progress
[frames] | no frames]

Source Code for Module buildbot.status.progress

  1  # This file is part of Buildbot.  Buildbot is free software: you can 
  2  # redistribute it and/or modify it under the terms of the GNU General Public 
  3  # License as published by the Free Software Foundation, version 2. 
  4  # 
  5  # This program is distributed in the hope that it will be useful, but WITHOUT 
  6  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
  7  # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more 
  8  # details. 
  9  # 
 10  # You should have received a copy of the GNU General Public License along with 
 11  # this program; if not, write to the Free Software Foundation, Inc., 51 
 12  # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 
 13  # 
 14  # Copyright Buildbot Team Members 
 15   
 16   
 17  from twisted.internet import reactor 
 18  from twisted.spread import pb 
 19  from twisted.python import log 
 20  from buildbot import util 
 21   
22 -class StepProgress:
23 """I keep track of how much progress a single BuildStep has made. 24 25 Progress is measured along various axes. Time consumed is one that is 26 available for all steps. Amount of command output is another, and may be 27 better quantified by scanning the output for markers to derive number of 28 files compiled, directories walked, tests run, etc. 29 30 I am created when the build begins, and given to a BuildProgress object 31 so it can track the overall progress of the whole build. 32 33 """ 34 35 startTime = None 36 stopTime = None 37 expectedTime = None 38 buildProgress = None 39 debug = False 40
41 - def __init__(self, name, metricNames):
42 self.name = name 43 self.progress = {} 44 self.expectations = {} 45 for m in metricNames: 46 self.progress[m] = None 47 self.expectations[m] = None
48
49 - def setBuildProgress(self, bp):
50 self.buildProgress = bp
51
52 - def setExpectations(self, metrics):
53 """The step can call this to explicitly set a target value for one 54 of its metrics. E.g., ShellCommands knows how many commands it will 55 execute, so it could set the 'commands' expectation.""" 56 for metric, value in metrics.items(): 57 self.expectations[metric] = value 58 self.buildProgress.newExpectations()
59
60 - def setExpectedTime(self, seconds):
61 self.expectedTime = seconds 62 self.buildProgress.newExpectations()
63
64 - def start(self):
65 if self.debug: print "StepProgress.start[%s]" % self.name 66 self.startTime = util.now()
67
68 - def setProgress(self, metric, value):
69 """The step calls this as progress is made along various axes.""" 70 if self.debug: 71 print "setProgress[%s][%s] = %s" % (self.name, metric, value) 72 self.progress[metric] = value 73 if self.debug: 74 r = self.remaining() 75 print " step remaining:", r 76 self.buildProgress.newProgress()
77
78 - def finish(self):
79 """This stops the 'time' metric and marks the step as finished 80 overall. It should be called after the last .setProgress has been 81 done for each axis.""" 82 if self.debug: print "StepProgress.finish[%s]" % self.name 83 self.stopTime = util.now() 84 self.buildProgress.stepFinished(self.name)
85
86 - def totalTime(self):
87 if self.startTime != None and self.stopTime != None: 88 return self.stopTime - self.startTime
89
90 - def remaining(self):
91 if self.startTime == None: 92 return self.expectedTime 93 if self.stopTime != None: 94 return 0 # already finished 95 # TODO: replace this with cleverness that graphs each metric vs. 96 # time, then finds the inverse function. Will probably need to save 97 # a timestamp with each setProgress update, when finished, go back 98 # and find the 2% transition points, then save those 50 values in a 99 # list. On the next build, do linear interpolation between the two 100 # closest samples to come up with a percentage represented by that 101 # metric. 102 103 # TODO: If no other metrics are available, just go with elapsed 104 # time. Given the non-time-uniformity of text output from most 105 # steps, this would probably be better than the text-percentage 106 # scheme currently implemented. 107 108 percentages = [] 109 for metric, value in self.progress.items(): 110 expectation = self.expectations[metric] 111 if value != None and expectation != None: 112 p = 1.0 * value / expectation 113 percentages.append(p) 114 if percentages: 115 avg = reduce(lambda x,y: x+y, percentages) / len(percentages) 116 if avg > 1.0: 117 # overdue 118 avg = 1.0 119 if avg < 0.0: 120 avg = 0.0 121 if percentages and self.expectedTime != None: 122 return self.expectedTime - (avg * self.expectedTime) 123 if self.expectedTime is not None: 124 # fall back to pure time 125 return self.expectedTime - (util.now() - self.startTime) 126 return None # no idea
127 128
129 -class WatcherState:
130 - def __init__(self, interval):
131 self.interval = interval 132 self.timer = None 133 self.needUpdate = 0
134
135 -class BuildProgress(pb.Referenceable):
136 """I keep track of overall build progress. I hold a list of StepProgress 137 objects. 138 """ 139
140 - def __init__(self, stepProgresses):
141 self.steps = {} 142 for s in stepProgresses: 143 self.steps[s.name] = s 144 s.setBuildProgress(self) 145 self.finishedSteps = [] 146 self.watchers = {} 147 self.debug = 0
148
149 - def setExpectationsFrom(self, exp):
150 """Set our expectations from the builder's Expectations object.""" 151 for name, metrics in exp.steps.items(): 152 s = self.steps[name] 153 s.setExpectedTime(exp.times[name]) 154 s.setExpectations(exp.steps[name])
155
156 - def newExpectations(self):
157 """Call this when one of the steps has changed its expectations. 158 This should trigger us to update our ETA value and notify any 159 subscribers.""" 160 pass # subscribers are not implemented: they just poll
161
162 - def stepFinished(self, stepname):
163 assert(stepname not in self.finishedSteps) 164 self.finishedSteps.append(stepname) 165 if len(self.finishedSteps) == len(self.steps.keys()): 166 self.sendLastUpdates()
167
168 - def newProgress(self):
169 r = self.remaining() 170 if self.debug: 171 print " remaining:", r 172 if r != None: 173 self.sendAllUpdates()
174
175 - def remaining(self):
176 # sum eta of all steps 177 sum = 0 178 for name, step in self.steps.items(): 179 rem = step.remaining() 180 if rem == None: 181 return None # not sure 182 sum += rem 183 return sum
184 - def eta(self):
185 left = self.remaining() 186 if left == None: 187 return None # not sure 188 done = util.now() + left 189 return done
190 191
192 - def remote_subscribe(self, remote, interval=5):
193 # [interval, timer, needUpdate] 194 # don't send an update more than once per interval 195 self.watchers[remote] = WatcherState(interval) 196 remote.notifyOnDisconnect(self.removeWatcher) 197 self.updateWatcher(remote) 198 self.startTimer(remote) 199 log.msg("BuildProgress.remote_subscribe(%s)" % remote)
200 - def remote_unsubscribe(self, remote):
201 # TODO: this doesn't work. I think 'remote' will always be different 202 # than the object that appeared in _subscribe. 203 log.msg("BuildProgress.remote_unsubscribe(%s)" % remote) 204 self.removeWatcher(remote)
205 #remote.dontNotifyOnDisconnect(self.removeWatcher)
206 - def removeWatcher(self, remote):
207 #log.msg("removeWatcher(%s)" % remote) 208 try: 209 timer = self.watchers[remote].timer 210 if timer: 211 timer.cancel() 212 del self.watchers[remote] 213 except KeyError: 214 log.msg("Weird, removeWatcher on non-existent subscriber:", 215 remote)
216 - def sendAllUpdates(self):
217 for r in self.watchers.keys(): 218 self.updateWatcher(r)
219 - def updateWatcher(self, remote):
220 # an update wants to go to this watcher. Send it if we can, otherwise 221 # queue it for later 222 w = self.watchers[remote] 223 if not w.timer: 224 # no timer, so send update now and start the timer 225 self.sendUpdate(remote) 226 self.startTimer(remote) 227 else: 228 # timer is running, just mark as needing an update 229 w.needUpdate = 1
230 - def startTimer(self, remote):
231 w = self.watchers[remote] 232 timer = reactor.callLater(w.interval, self.watcherTimeout, remote) 233 w.timer = timer
234 - def sendUpdate(self, remote, last=0):
235 self.watchers[remote].needUpdate = 0 236 #text = self.asText() # TODO: not text, duh 237 try: 238 remote.callRemote("progress", self.remaining()) 239 if last: 240 remote.callRemote("finished", self) 241 except: 242 log.deferr() 243 self.removeWatcher(remote)
244
245 - def watcherTimeout(self, remote):
246 w = self.watchers.get(remote, None) 247 if not w: 248 return # went away 249 w.timer = None 250 if w.needUpdate: 251 self.sendUpdate(remote) 252 self.startTimer(remote)
253 - def sendLastUpdates(self):
254 for remote in self.watchers.keys(): 255 self.sendUpdate(remote, 1) 256 self.removeWatcher(remote)
257 258
259 -class Expectations:
260 debug = False 261 # decay=1.0 ignores all but the last build 262 # 0.9 is short time constant. 0.1 is very long time constant 263 # TODO: let decay be specified per-metric 264 decay = 0.5 265
266 - def __init__(self, buildprogress):
267 """Create us from a successful build. We will expect each step to 268 take as long as it did in that build.""" 269 270 # .steps maps stepname to dict2 271 # dict2 maps metricname to final end-of-step value 272 self.steps = {} 273 274 # .times maps stepname to per-step elapsed time 275 self.times = {} 276 277 for name, step in buildprogress.steps.items(): 278 self.steps[name] = {} 279 for metric, value in step.progress.items(): 280 self.steps[name][metric] = value 281 self.times[name] = None 282 if step.startTime is not None and step.stopTime is not None: 283 self.times[name] = step.stopTime - step.startTime
284
285 - def wavg(self, old, current):
286 if old is None: 287 return current 288 if current is None: 289 return old 290 else: 291 return (current * self.decay) + (old * (1 - self.decay))
292
293 - def update(self, buildprogress):
294 for name, stepprogress in buildprogress.steps.items(): 295 old = self.times[name] 296 current = stepprogress.totalTime() 297 if current == None: 298 log.msg("Expectations.update: current[%s] was None!" % name) 299 continue 300 new = self.wavg(old, current) 301 self.times[name] = new 302 if self.debug: 303 print "new expected time[%s] = %s, old %s, cur %s" % \ 304 (name, new, old, current) 305 306 for metric, current in stepprogress.progress.items(): 307 old = self.steps[name][metric] 308 new = self.wavg(old, current) 309 if self.debug: 310 print "new expectation[%s][%s] = %s, old %s, cur %s" % \ 311 (name, metric, new, old, current) 312 self.steps[name][metric] = new
313
314 - def expectedBuildTime(self):
315 if None in self.times.values(): 316 return None 317 #return sum(self.times.values()) 318 # python-2.2 doesn't have 'sum'. TODO: drop python-2.2 support 319 s = 0 320 for v in self.times.values(): 321 s += v 322 return s
323