buildbot.changes.bonsaipoller

1 import time 2 from xml.dom import minidom 3 4 from twisted.python import log, failure 5 from twisted.internet import reactor 6 from twisted.internet.task import LoopingCall 7 from twisted.web.client import getPage 8 9 from buildbot.changes import base, changes 10

11 -class InvalidResultError(Exception):

12 - def __init__(self, value="InvalidResultError"):

13 self.value = value

14 - def __str__(self):

15 return repr(self.value)

16

17 -class EmptyResult(Exception):

18 pass

19

20 -class NoMoreCiNodes(Exception):

21 pass

22

23 -class NoMoreFileNodes(Exception):

24 pass

25

26 -class BonsaiResult:

27 """I hold a list of CiNodes"""

28 - def __init__(self, nodes=[]):

29 self.nodes = nodes

30

31 - def __cmp__(self, other):

32 if len(self.nodes) != len(other.nodes): 33 return False 34 for i in range(len(self.nodes)): 35 if self.nodes[i].log != other.nodes[i].log \ 36 or self.nodes[i].who != other.nodes[i].who \ 37 or self.nodes[i].date != other.nodes[i].date \ 38 or len(self.nodes[i].files) != len(other.nodes[i].files): 39 return -1 40 41 for j in range(len(self.nodes[i].files)): 42 if self.nodes[i].files[j].revision \ 43 != other.nodes[i].files[j].revision \ 44 or self.nodes[i].files[j].filename \ 45 != other.nodes[i].files[j].filename: 46 return -1 47 48 return 0

49

50 -class CiNode:

51 """I hold information baout one <ci> node, including a list of files"""

52 - def __init__(self, log="", who="", date=0, files=[]):

53 self.log = log 54 self.who = who 55 self.date = date 56 self.files = files

57

58 -class FileNode:

59 """I hold information about one <f> node"""

60 - def __init__(self, revision="", filename=""):

61 self.revision = revision 62 self.filename = filename

63

64 -class BonsaiParser:

65 """I parse the XML result from a bonsai cvsquery.""" 66

67 - def __init__(self, data):

68 try: 69 # this is a fix for non-ascii characters 70 # because bonsai does not give us an encoding to work with 71 # it impossible to be 100% sure what to decode it as but latin1 covers 72 # the broadest base 73 data = data.decode("latin1") 74 data = data.encode("ascii", "replace") 75 self.dom = minidom.parseString(data) 76 log.msg(data) 77 except: 78 raise InvalidResultError("Malformed XML in result") 79 80 self.ciNodes = self.dom.getElementsByTagName("ci") 81 self.currentCiNode = None # filled in by _nextCiNode() 82 self.fileNodes = None # filled in by _nextCiNode() 83 self.currentFileNode = None # filled in by _nextFileNode() 84 self.bonsaiResult = self._parseData()

85

86 - def getData(self):

87 return self.bonsaiResult

88

89 - def _parseData(self):

90 """Returns data from a Bonsai cvsquery in a BonsaiResult object""" 91 nodes = [] 92 try: 93 while self._nextCiNode(): 94 files = [] 95 try: 96 while self._nextFileNode(): 97 files.append(FileNode(self._getRevision(), 98 self._getFilename())) 99 except NoMoreFileNodes: 100 pass 101 except InvalidResultError: 102 raise 103 cinode = CiNode(self._getLog(), self._getWho(), 104 self._getDate(), files) 105 # hack around bonsai xml output bug for empty check-in comments 106 if not cinode.log and nodes and \ 107 not nodes[-1].log and \ 108 cinode.who == nodes[-1].who and \ 109 cinode.date == nodes[-1].date: 110 nodes[-1].files += cinode.files 111 else: 112 nodes.append(cinode) 113 114 except NoMoreCiNodes: 115 pass 116 except (InvalidResultError, EmptyResult): 117 raise 118 119 return BonsaiResult(nodes)

120 121

122 - def _nextCiNode(self):

123 """Iterates to the next <ci> node and fills self.fileNodes with 124 child <f> nodes""" 125 try: 126 self.currentCiNode = self.ciNodes.pop(0) 127 if len(self.currentCiNode.getElementsByTagName("files")) > 1: 128 raise InvalidResultError("Multiple <files> for one <ci>") 129 130 self.fileNodes = self.currentCiNode.getElementsByTagName("f") 131 except IndexError: 132 # if there was zero <ci> nodes in the result 133 if not self.currentCiNode: 134 raise EmptyResult 135 else: 136 raise NoMoreCiNodes 137 138 return True

139

140 - def _nextFileNode(self):

141 """Iterates to the next <f> node""" 142 try: 143 self.currentFileNode = self.fileNodes.pop(0) 144 except IndexError: 145 raise NoMoreFileNodes 146 147 return True

148

149 - def _getLog(self):

150 """Returns the log of the current <ci> node""" 151 logs = self.currentCiNode.getElementsByTagName("log") 152 if len(logs) < 1: 153 raise InvalidResultError("No log present") 154 elif len(logs) > 1: 155 raise InvalidResultError("Multiple logs present") 156 157 # catch empty check-in comments 158 if logs[0].firstChild: 159 return logs[0].firstChild.data 160 return ''

161

162 - def _getWho(self):

163 """Returns the e-mail address of the commiter""" 164 # convert unicode string to regular string 165 return str(self.currentCiNode.getAttribute("who"))

166

167 - def _getDate(self):

168 """Returns the date (unix time) of the commit""" 169 # convert unicode number to regular one 170 try: 171 commitDate = int(self.currentCiNode.getAttribute("date")) 172 except ValueError: 173 raise InvalidResultError 174 175 return commitDate

176

177 - def _getFilename(self):

178 """Returns the filename of the current <f> node""" 179 try: 180 filename = self.currentFileNode.firstChild.data 181 except AttributeError: 182 raise InvalidResultError("Missing filename") 183 184 return filename

185

186 - def _getRevision(self):

187 return self.currentFileNode.getAttribute("rev")

188 189

190 -class BonsaiPoller(base.ChangeSource):

191 """This source will poll a bonsai server for changes and submit 192 them to the change master.""" 193 194 compare_attrs = ["bonsaiURL", "pollInterval", "tree", 195 "module", "branch", "cvsroot"] 196 197 parent = None # filled in when we're added 198 loop = None 199 volatile = ['loop'] 200 working = False 201

202 - def __init__(self, bonsaiURL, module, branch, tree="default", 203 cvsroot="/cvsroot", pollInterval=30, project=''):

204 """ 205 @type bonsaiURL: string 206 @param bonsaiURL: The base URL of the Bonsai server 207 (ie. http://bonsai.mozilla.org) 208 @type module: string 209 @param module: The module to look for changes in. Commonly 210 this is 'all' 211 @type branch: string 212 @param branch: The branch to look for changes in. This must 213 match the 214 'branch' option for the Scheduler. 215 @type tree: string 216 @param tree: The tree to look for changes in. Commonly this 217 is 'all' 218 @type cvsroot: string 219 @param cvsroot: The cvsroot of the repository. Usually this is 220 '/cvsroot' 221 @type pollInterval: int 222 @param pollInterval: The time (in seconds) between queries for 223 changes 224 225 @type project: string 226 @param project: project to attach to all Changes from this changesource 227 """ 228 229 self.bonsaiURL = bonsaiURL 230 self.module = module 231 self.branch = branch 232 self.tree = tree 233 self.cvsroot = cvsroot 234 self.repository = module != 'all' and module or '' 235 self.pollInterval = pollInterval 236 self.lastChange = time.time() 237 self.lastPoll = time.time()

238

239 - def startService(self):

240 self.loop = LoopingCall(self.poll) 241 base.ChangeSource.startService(self) 242 243 reactor.callLater(0, self.loop.start, self.pollInterval)

244

245 - def stopService(self):

246 self.loop.stop() 247 return base.ChangeSource.stopService(self)

248

249 - def describe(self):

250 str = "" 251 str += "Getting changes from the Bonsai service running at %s " \ 252 % self.bonsaiURL 253 str += "<br>Using tree: %s, branch: %s, and module: %s" % (self.tree, \ 254 self.branch, self.module) 255 return str

256

257 - def poll(self):

258 if self.working: 259 log.msg("Not polling Bonsai because last poll is still working") 260 else: 261 self.working = True 262 d = self._get_changes() 263 d.addCallback(self._process_changes) 264 d.addCallbacks(self._finished_ok, self._finished_failure) 265 return

266

267 - def _finished_ok(self, res):

268 assert self.working 269 self.working = False 270 271 # check for failure -- this is probably never hit but the twisted docs 272 # are not clear enough to be sure. it is being kept "just in case" 273 if isinstance(res, failure.Failure): 274 log.msg("Bonsai poll failed: %s" % res) 275 return res

276

277 - def _finished_failure(self, res):

278 log.msg("Bonsai poll failed: %s" % res) 279 assert self.working 280 self.working = False 281 return None # eat the failure

282

283 - def _make_url(self):

284 args = ["treeid=%s" % self.tree, "module=%s" % self.module, 285 "branch=%s" % self.branch, "branchtype=match", 286 "sortby=Date", "date=explicit", 287 "mindate=%d" % self.lastChange, 288 "maxdate=%d" % int(time.time()), 289 "cvsroot=%s" % self.cvsroot, "xml=1"] 290 # build the bonsai URL 291 url = self.bonsaiURL 292 url += "/cvsquery.cgi?" 293 url += "&".join(args) 294 295 return url

296

297 - def _get_changes(self):

298 url = self._make_url() 299 log.msg("Polling Bonsai tree at %s" % url) 300 301 self.lastPoll = time.time() 302 # get the page, in XML format 303 return getPage(url, timeout=self.pollInterval)

304

305 - def _process_changes(self, query):

306 try: 307 bp = BonsaiParser(query) 308 result = bp.getData() 309 except InvalidResultError, e: 310 log.msg("Could not process Bonsai query: " + e.value) 311 return 312 except EmptyResult: 313 return 314 315 for cinode in result.nodes: 316 files = [file.filename + ' (revision '+file.revision+')' 317 for file in cinode.files] 318 c = changes.Change(who = cinode.who, 319 files = files, 320 comments = cinode.log, 321 when = cinode.date, 322 branch = self.branch) 323 self.parent.addChange(c) 324 self.lastChange = self.lastPoll

325

Source Code for Module buildbot.changes.bonsaipoller