Package buildbot :: Package changes :: Module bonsaipoller
[frames] | no frames]

Source Code for Module buildbot.changes.bonsaipoller

  1  import time 
  2  from xml.dom import minidom 
  3   
  4  from twisted.python import log, failure 
  5  from twisted.internet import reactor 
  6  from twisted.internet.task import LoopingCall 
  7  from twisted.web.client import getPage 
  8   
  9  from buildbot.changes import base, changes 
 10   
11 -class InvalidResultError(Exception):
12 - def __init__(self, value="InvalidResultError"):
13 self.value = value
14 - def __str__(self):
15 return repr(self.value)
16
17 -class EmptyResult(Exception):
18 pass
19
20 -class NoMoreCiNodes(Exception):
21 pass
22
23 -class NoMoreFileNodes(Exception):
24 pass
25
26 -class BonsaiResult:
27 """I hold a list of CiNodes"""
28 - def __init__(self, nodes=[]):
29 self.nodes = nodes
30
31 - def __cmp__(self, other):
32 if len(self.nodes) != len(other.nodes): 33 return False 34 for i in range(len(self.nodes)): 35 if self.nodes[i].log != other.nodes[i].log \ 36 or self.nodes[i].who != other.nodes[i].who \ 37 or self.nodes[i].date != other.nodes[i].date \ 38 or len(self.nodes[i].files) != len(other.nodes[i].files): 39 return -1 40 41 for j in range(len(self.nodes[i].files)): 42 if self.nodes[i].files[j].revision \ 43 != other.nodes[i].files[j].revision \ 44 or self.nodes[i].files[j].filename \ 45 != other.nodes[i].files[j].filename: 46 return -1 47 48 return 0
49
50 -class CiNode:
51 """I hold information baout one <ci> node, including a list of files"""
52 - def __init__(self, log="", who="", date=0, files=[]):
53 self.log = log 54 self.who = who 55 self.date = date 56 self.files = files
57
58 -class FileNode:
59 """I hold information about one <f> node"""
60 - def __init__(self, revision="", filename=""):
61 self.revision = revision 62 self.filename = filename
63
64 -class BonsaiParser:
65 """I parse the XML result from a bonsai cvsquery.""" 66
67 - def __init__(self, data):
68 try: 69 # this is a fix for non-ascii characters 70 # because bonsai does not give us an encoding to work with 71 # it impossible to be 100% sure what to decode it as but latin1 covers 72 # the broadest base 73 data = data.decode("latin1") 74 data = data.encode("ascii", "replace") 75 self.dom = minidom.parseString(data) 76 log.msg(data) 77 except: 78 raise InvalidResultError("Malformed XML in result") 79 80 self.ciNodes = self.dom.getElementsByTagName("ci") 81 self.currentCiNode = None # filled in by _nextCiNode() 82 self.fileNodes = None # filled in by _nextCiNode() 83 self.currentFileNode = None # filled in by _nextFileNode() 84 self.bonsaiResult = self._parseData()
85
86 - def getData(self):
87 return self.bonsaiResult
88
89 - def _parseData(self):
90 """Returns data from a Bonsai cvsquery in a BonsaiResult object""" 91 nodes = [] 92 try: 93 while self._nextCiNode(): 94 files = [] 95 try: 96 while self._nextFileNode(): 97 files.append(FileNode(self._getRevision(), 98 self._getFilename())) 99 except NoMoreFileNodes: 100 pass 101 except InvalidResultError: 102 raise 103 cinode = CiNode(self._getLog(), self._getWho(), 104 self._getDate(), files) 105 # hack around bonsai xml output bug for empty check-in comments 106 if not cinode.log and nodes and \ 107 not nodes[-1].log and \ 108 cinode.who == nodes[-1].who and \ 109 cinode.date == nodes[-1].date: 110 nodes[-1].files += cinode.files 111 else: 112 nodes.append(cinode) 113 114 except NoMoreCiNodes: 115 pass 116 except (InvalidResultError, EmptyResult): 117 raise 118 119 return BonsaiResult(nodes)
120 121
122 - def _nextCiNode(self):
123 """Iterates to the next <ci> node and fills self.fileNodes with 124 child <f> nodes""" 125 try: 126 self.currentCiNode = self.ciNodes.pop(0) 127 if len(self.currentCiNode.getElementsByTagName("files")) > 1: 128 raise InvalidResultError("Multiple <files> for one <ci>") 129 130 self.fileNodes = self.currentCiNode.getElementsByTagName("f") 131 except IndexError: 132 # if there was zero <ci> nodes in the result 133 if not self.currentCiNode: 134 raise EmptyResult 135 else: 136 raise NoMoreCiNodes 137 138 return True
139
140 - def _nextFileNode(self):
141 """Iterates to the next <f> node""" 142 try: 143 self.currentFileNode = self.fileNodes.pop(0) 144 except IndexError: 145 raise NoMoreFileNodes 146 147 return True
148
149 - def _getLog(self):
150 """Returns the log of the current <ci> node""" 151 logs = self.currentCiNode.getElementsByTagName("log") 152 if len(logs) < 1: 153 raise InvalidResultError("No log present") 154 elif len(logs) > 1: 155 raise InvalidResultError("Multiple logs present") 156 157 # catch empty check-in comments 158 if logs[0].firstChild: 159 return logs[0].firstChild.data 160 return ''
161
162 - def _getWho(self):
163 """Returns the e-mail address of the commiter""" 164 # convert unicode string to regular string 165 return str(self.currentCiNode.getAttribute("who"))
166
167 - def _getDate(self):
168 """Returns the date (unix time) of the commit""" 169 # convert unicode number to regular one 170 try: 171 commitDate = int(self.currentCiNode.getAttribute("date")) 172 except ValueError: 173 raise InvalidResultError 174 175 return commitDate
176
177 - def _getFilename(self):
178 """Returns the filename of the current <f> node""" 179 try: 180 filename = self.currentFileNode.firstChild.data 181 except AttributeError: 182 raise InvalidResultError("Missing filename") 183 184 return filename
185
186 - def _getRevision(self):
187 return self.currentFileNode.getAttribute("rev")
188 189
190 -class BonsaiPoller(base.ChangeSource):
191 """This source will poll a bonsai server for changes and submit 192 them to the change master.""" 193 194 compare_attrs = ["bonsaiURL", "pollInterval", "tree", 195 "module", "branch", "cvsroot"] 196 197 parent = None # filled in when we're added 198 loop = None 199 volatile = ['loop'] 200 working = False 201
202 - def __init__(self, bonsaiURL, module, branch, tree="default", 203 cvsroot="/cvsroot", pollInterval=30, project=''):
204 """ 205 @type bonsaiURL: string 206 @param bonsaiURL: The base URL of the Bonsai server 207 (ie. http://bonsai.mozilla.org) 208 @type module: string 209 @param module: The module to look for changes in. Commonly 210 this is 'all' 211 @type branch: string 212 @param branch: The branch to look for changes in. This must 213 match the 214 'branch' option for the Scheduler. 215 @type tree: string 216 @param tree: The tree to look for changes in. Commonly this 217 is 'all' 218 @type cvsroot: string 219 @param cvsroot: The cvsroot of the repository. Usually this is 220 '/cvsroot' 221 @type pollInterval: int 222 @param pollInterval: The time (in seconds) between queries for 223 changes 224 225 @type project: string 226 @param project: project to attach to all Changes from this changesource 227 """ 228 229 self.bonsaiURL = bonsaiURL 230 self.module = module 231 self.branch = branch 232 self.tree = tree 233 self.cvsroot = cvsroot 234 self.repository = module != 'all' and module or '' 235 self.pollInterval = pollInterval 236 self.lastChange = time.time() 237 self.lastPoll = time.time()
238
239 - def startService(self):
240 self.loop = LoopingCall(self.poll) 241 base.ChangeSource.startService(self) 242 243 reactor.callLater(0, self.loop.start, self.pollInterval)
244
245 - def stopService(self):
246 self.loop.stop() 247 return base.ChangeSource.stopService(self)
248
249 - def describe(self):
250 str = "" 251 str += "Getting changes from the Bonsai service running at %s " \ 252 % self.bonsaiURL 253 str += "<br>Using tree: %s, branch: %s, and module: %s" % (self.tree, \ 254 self.branch, self.module) 255 return str
256
257 - def poll(self):
258 if self.working: 259 log.msg("Not polling Bonsai because last poll is still working") 260 else: 261 self.working = True 262 d = self._get_changes() 263 d.addCallback(self._process_changes) 264 d.addCallbacks(self._finished_ok, self._finished_failure) 265 return
266
267 - def _finished_ok(self, res):
268 assert self.working 269 self.working = False 270 271 # check for failure -- this is probably never hit but the twisted docs 272 # are not clear enough to be sure. it is being kept "just in case" 273 if isinstance(res, failure.Failure): 274 log.msg("Bonsai poll failed: %s" % res) 275 return res
276
277 - def _finished_failure(self, res):
278 log.msg("Bonsai poll failed: %s" % res) 279 assert self.working 280 self.working = False 281 return None # eat the failure
282
283 - def _make_url(self):
284 args = ["treeid=%s" % self.tree, "module=%s" % self.module, 285 "branch=%s" % self.branch, "branchtype=match", 286 "sortby=Date", "date=explicit", 287 "mindate=%d" % self.lastChange, 288 "maxdate=%d" % int(time.time()), 289 "cvsroot=%s" % self.cvsroot, "xml=1"] 290 # build the bonsai URL 291 url = self.bonsaiURL 292 url += "/cvsquery.cgi?" 293 url += "&".join(args) 294 295 return url
296
297 - def _get_changes(self):
298 url = self._make_url() 299 log.msg("Polling Bonsai tree at %s" % url) 300 301 self.lastPoll = time.time() 302 # get the page, in XML format 303 return getPage(url, timeout=self.pollInterval)
304
305 - def _process_changes(self, query):
306 try: 307 bp = BonsaiParser(query) 308 result = bp.getData() 309 except InvalidResultError, e: 310 log.msg("Could not process Bonsai query: " + e.value) 311 return 312 except EmptyResult: 313 return 314 315 for cinode in result.nodes: 316 files = [file.filename + ' (revision '+file.revision+')' 317 for file in cinode.files] 318 c = changes.Change(who = cinode.who, 319 files = files, 320 comments = cinode.log, 321 when = cinode.date, 322 branch = self.branch) 323 self.parent.addChange(c) 324 self.lastChange = self.lastPoll
325