buildbot.changes.bonsaipoller

1 import time 2 from xml.dom import minidom 3 4 from twisted.python import log, failure 5 from twisted.internet import reactor 6 from twisted.internet.task import LoopingCall 7 from twisted.web.client import getPage 8 9 from buildbot.changes import base, changes 10

11 -class InvalidResultError(Exception):

12 - def __init__(self, value="InvalidResultError"):

13 self.value = value

14 - def __str__(self):

15 return repr(self.value)

16

17 -class EmptyResult(Exception):

18 pass

19

20 -class NoMoreCiNodes(Exception):

21 pass

22

23 -class NoMoreFileNodes(Exception):

24 pass

25

26 -class BonsaiResult:

27 """I hold a list of CiNodes"""

28 - def __init__(self, nodes=[]):

29 self.nodes = nodes

30

31 - def __cmp__(self, other):

32 if len(self.nodes) != len(other.nodes): 33 return False 34 for i in range(len(self.nodes)): 35 if self.nodes[i].log != other.nodes[i].log \ 36 or self.nodes[i].who != other.nodes[i].who \ 37 or self.nodes[i].date != other.nodes[i].date \ 38 or len(self.nodes[i].files) != len(other.nodes[i].files): 39 return -1 40 41 for j in range(len(self.nodes[i].files)): 42 if self.nodes[i].files[j].revision \ 43 != other.nodes[i].files[j].revision \ 44 or self.nodes[i].files[j].filename \ 45 != other.nodes[i].files[j].filename: 46 return -1 47 48 return 0

49

50 -class CiNode:

51 """I hold information baout one <ci> node, including a list of files"""

52 - def __init__(self, log="", who="", date=0, files=[]):

53 self.log = log 54 self.who = who 55 self.date = date 56 self.files = files

57

58 -class FileNode:

59 """I hold information about one <f> node"""

60 - def __init__(self, revision="", filename=""):

61 self.revision = revision 62 self.filename = filename

63

64 -class BonsaiParser:

65 """I parse the XML result from a bonsai cvsquery.""" 66

67 - def __init__(self, data):

68 try: 69 # this is a fix for non-ascii characters 70 # because bonsai does not give us an encoding to work with 71 # it impossible to be 100% sure what to decode it as but latin1 covers 72 # the broadest base 73 data = data.decode("latin1") 74 data = data.encode("ascii", "replace") 75 self.dom = minidom.parseString(data) 76 log.msg(data) 77 except: 78 raise InvalidResultError("Malformed XML in result") 79 80 self.ciNodes = self.dom.getElementsByTagName("ci") 81 self.currentCiNode = None # filled in by _nextCiNode() 82 self.fileNodes = None # filled in by _nextCiNode() 83 self.currentFileNode = None # filled in by _nextFileNode() 84 self.bonsaiResult = self._parseData()

85

86 - def getData(self):

87 return self.bonsaiResult

88

89 - def _parseData(self):

90 """Returns data from a Bonsai cvsquery in a BonsaiResult object""" 91 nodes = [] 92 try: 93 while self._nextCiNode(): 94 files = [] 95 try: 96 while self._nextFileNode(): 97 files.append(FileNode(self._getRevision(), 98 self._getFilename())) 99 except NoMoreFileNodes: 100 pass 101 except InvalidResultError: 102 raise 103 cinode = CiNode(self._getLog(), self._getWho(), 104 self._getDate(), files) 105 # hack around bonsai xml output bug for empty check-in comments 106 if not cinode.log and nodes and \ 107 not nodes[-1].log and \ 108 cinode.who == nodes[-1].who and \ 109 cinode.date == nodes[-1].date: 110 nodes[-1].files += cinode.files 111 else: 112 nodes.append(cinode) 113 114 except NoMoreCiNodes: 115 pass 116 except InvalidResultError, EmptyResult: 117 raise 118 119 return BonsaiResult(nodes)

120 121

122 - def _nextCiNode(self):

123 """Iterates to the next <ci> node and fills self.fileNodes with 124 child <f> nodes""" 125 try: 126 self.currentCiNode = self.ciNodes.pop(0) 127 if len(self.currentCiNode.getElementsByTagName("files")) > 1: 128 raise InvalidResultError("Multiple <files> for one <ci>") 129 130 self.fileNodes = self.currentCiNode.getElementsByTagName("f") 131 except IndexError: 132 # if there was zero <ci> nodes in the result 133 if not self.currentCiNode: 134 raise EmptyResult 135 else: 136 raise NoMoreCiNodes 137 138 return True

139

140 - def _nextFileNode(self):

141 """Iterates to the next <f> node""" 142 try: 143 self.currentFileNode = self.fileNodes.pop(0) 144 except IndexError: 145 raise NoMoreFileNodes 146 147 return True

148

149 - def _getLog(self):

150 """Returns the log of the current <ci> node""" 151 logs = self.currentCiNode.getElementsByTagName("log") 152 if len(logs) < 1: 153 raise InvalidResultError("No log present") 154 elif len(logs) > 1: 155 raise InvalidResultError("Multiple logs present") 156 157 # catch empty check-in comments 158 if logs[0].firstChild: 159 return logs[0].firstChild.data 160 return ''

161

162 - def _getWho(self):

163 """Returns the e-mail address of the commiter""" 164 # convert unicode string to regular string 165 return str(self.currentCiNode.getAttribute("who"))

166

167 - def _getDate(self):

168 """Returns the date (unix time) of the commit""" 169 # convert unicode number to regular one 170 try: 171 commitDate = int(self.currentCiNode.getAttribute("date")) 172 except ValueError: 173 raise InvalidResultError 174 175 return commitDate

176

177 - def _getFilename(self):

178 """Returns the filename of the current <f> node""" 179 try: 180 filename = self.currentFileNode.firstChild.data 181 except AttributeError: 182 raise InvalidResultError("Missing filename") 183 184 return filename

185

186 - def _getRevision(self):

187 return self.currentFileNode.getAttribute("rev")

188 189

190 -class BonsaiPoller(base.ChangeSource):

191 """This source will poll a bonsai server for changes and submit 192 them to the change master.""" 193 194 compare_attrs = ["bonsaiURL", "pollInterval", "tree", 195 "module", "branch", "cvsroot"] 196 197 parent = None # filled in when we're added 198 loop = None 199 volatile = ['loop'] 200 working = False 201

202 - def __init__(self, bonsaiURL, module, branch, tree="default", 203 cvsroot="/cvsroot", pollInterval=30):

204 """ 205 @type bonsaiURL: string 206 @param bonsaiURL: The base URL of the Bonsai server 207 (ie. http://bonsai.mozilla.org) 208 @type module: string 209 @param module: The module to look for changes in. Commonly 210 this is 'all' 211 @type branch: string 212 @param branch: The branch to look for changes in. This must 213 match the 214 'branch' option for the Scheduler. 215 @type tree: string 216 @param tree: The tree to look for changes in. Commonly this 217 is 'all' 218 @type cvsroot: string 219 @param cvsroot: The cvsroot of the repository. Usually this is 220 '/cvsroot' 221 @type pollInterval: int 222 @param pollInterval: The time (in seconds) between queries for 223 changes 224 """ 225 226 self.bonsaiURL = bonsaiURL 227 self.module = module 228 self.branch = branch 229 self.tree = tree 230 self.cvsroot = cvsroot 231 self.pollInterval = pollInterval 232 self.lastChange = time.time() 233 self.lastPoll = time.time()

234

235 - def startService(self):

236 self.loop = LoopingCall(self.poll) 237 base.ChangeSource.startService(self) 238 239 reactor.callLater(0, self.loop.start, self.pollInterval)

240

241 - def stopService(self):

242 self.loop.stop() 243 return base.ChangeSource.stopService(self)

244

245 - def describe(self):

246 str = "" 247 str += "Getting changes from the Bonsai service running at %s " \ 248 % self.bonsaiURL 249 str += "<br>Using tree: %s, branch: %s, and module: %s" % (self.tree, \ 250 self.branch, self.module) 251 return str

252

253 - def poll(self):

254 if self.working: 255 log.msg("Not polling Bonsai because last poll is still working") 256 else: 257 self.working = True 258 d = self._get_changes() 259 d.addCallback(self._process_changes) 260 d.addCallbacks(self._finished_ok, self._finished_failure) 261 return

262

263 - def _finished_ok(self, res):

264 assert self.working 265 self.working = False 266 267 # check for failure -- this is probably never hit but the twisted docs 268 # are not clear enough to be sure. it is being kept "just in case" 269 if isinstance(res, failure.Failure): 270 log.msg("Bonsai poll failed: %s" % res) 271 return res

272

273 - def _finished_failure(self, res):

274 log.msg("Bonsai poll failed: %s" % res) 275 assert self.working 276 self.working = False 277 return None # eat the failure

278

279 - def _make_url(self):

280 args = ["treeid=%s" % self.tree, "module=%s" % self.module, 281 "branch=%s" % self.branch, "branchtype=match", 282 "sortby=Date", "date=explicit", 283 "mindate=%d" % self.lastChange, 284 "maxdate=%d" % int(time.time()), 285 "cvsroot=%s" % self.cvsroot, "xml=1"] 286 # build the bonsai URL 287 url = self.bonsaiURL 288 url += "/cvsquery.cgi?" 289 url += "&".join(args) 290 291 return url

292

293 - def _get_changes(self):

294 url = self._make_url() 295 log.msg("Polling Bonsai tree at %s" % url) 296 297 self.lastPoll = time.time() 298 # get the page, in XML format 299 return getPage(url, timeout=self.pollInterval)

300

301 - def _process_changes(self, query):

302 try: 303 bp = BonsaiParser(query) 304 result = bp.getData() 305 except InvalidResultError, e: 306 log.msg("Could not process Bonsai query: " + e.value) 307 return 308 except EmptyResult: 309 return 310 311 for cinode in result.nodes: 312 files = [file.filename + ' (revision '+file.revision+')' 313 for file in cinode.files] 314 c = changes.Change(who = cinode.who, 315 files = files, 316 comments = cinode.log, 317 when = cinode.date, 318 branch = self.branch) 319 self.parent.addChange(c) 320 self.lastChange = self.lastPoll

321

Source Code for Module buildbot.changes.bonsaipoller