buildbot.changes.svnpoller

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 17 # Based on the work of Dave Peticolas for the P4poll 18 # Changed to svn (using xml.dom.minidom) by Niklaus Giger 19 # Hacked beyond recognition by Brian Warner 20 21 from twisted.python import log 22 from twisted.internet import defer, utils 23 24 from buildbot import util 25 from buildbot.changes import base 26 27 import xml.dom.minidom 28 import os, urllib

29 30 # these split_file_* functions are available for use as values to the 31 # split_file= argument. 32 -def split_file_alwaystrunk(path):

33 return (None, path)

34

35 -def split_file_branches(path):

36 # turn trunk/subdir/file.c into (None, "subdir/file.c") 37 # and branches/1.5.x/subdir/file.c into ("branches/1.5.x", "subdir/file.c") 38 pieces = path.split('/') 39 if pieces[0] == 'trunk': 40 return (None, '/'.join(pieces[1:])) 41 elif pieces[0] == 'branches': 42 return ('/'.join(pieces[0:2]), '/'.join(pieces[2:])) 43 else: 44 return None

45

46 47 -class SVNPoller(base.PollingChangeSource, util.ComparableMixin):

48 """ 49 Poll a Subversion repository for changes and submit them to the change 50 master. 51 """ 52 53 compare_attrs = ["svnurl", "split_file", 54 "svnuser", "svnpasswd", 55 "pollInterval", "histmax", 56 "svnbin", "category", "cachepath"] 57 58 parent = None # filled in when we're added 59 last_change = None 60 loop = None 61

62 - def __init__(self, svnurl, split_file=None, 63 svnuser=None, svnpasswd=None, 64 pollInterval=10*60, histmax=100, 65 svnbin='svn', revlinktmpl='', category=None, 66 project='', cachepath=None, pollinterval=-2):

67 # for backward compatibility; the parameter used to be spelled with 'i' 68 if pollinterval != -2: 69 pollInterval = pollinterval 70 71 if svnurl.endswith("/"): 72 svnurl = svnurl[:-1] # strip the trailing slash 73 self.svnurl = svnurl 74 self.split_file = split_file or split_file_alwaystrunk 75 self.svnuser = svnuser 76 self.svnpasswd = svnpasswd 77 78 self.revlinktmpl = revlinktmpl 79 80 self.environ = os.environ.copy() # include environment variables 81 # required for ssh-agent auth 82 83 self.svnbin = svnbin 84 self.pollInterval = pollInterval 85 self.histmax = histmax 86 self._prefix = None 87 self.category = category 88 self.project = project 89 90 self.cachepath = cachepath 91 if self.cachepath and os.path.exists(self.cachepath): 92 try: 93 f = open(self.cachepath, "r") 94 self.last_change = int(f.read().strip()) 95 log.msg("SVNPoller: SVNPoller(%s) setting last_change to %s" % (self.svnurl, self.last_change)) 96 f.close() 97 # try writing it, too 98 f = open(self.cachepath, "w") 99 f.write(str(self.last_change)) 100 f.close() 101 except: 102 self.cachepath = None 103 log.msg(("SVNPoller: SVNPoller(%s) cache file corrupt or unwriteable; " + 104 "skipping and not using") % self.svnurl) 105 log.err()

106

107 - def describe(self):

108 return "SVNPoller: watching %s" % self.svnurl

109

110 - def poll(self):

111 # Our return value is only used for unit testing. 112 113 # we need to figure out the repository root, so we can figure out 114 # repository-relative pathnames later. Each SVNURL is in the form 115 # (ROOT)/(PROJECT)/(BRANCH)/(FILEPATH), where (ROOT) is something 116 # like svn://svn.twistedmatrix.com/svn/Twisted (i.e. there is a 117 # physical repository at /svn/Twisted on that host), (PROJECT) is 118 # something like Projects/Twisted (i.e. within the repository's 119 # internal namespace, everything under Projects/Twisted/ has 120 # something to do with Twisted, but these directory names do not 121 # actually appear on the repository host), (BRANCH) is something like 122 # "trunk" or "branches/2.0.x", and (FILEPATH) is a tree-relative 123 # filename like "twisted/internet/defer.py". 124 125 # our self.svnurl attribute contains (ROOT)/(PROJECT) combined 126 # together in a way that we can't separate without svn's help. If the 127 # user is not using the split_file= argument, then self.svnurl might 128 # be (ROOT)/(PROJECT)/(BRANCH) . In any case, the filenames we will 129 # get back from 'svn log' will be of the form 130 # (PROJECT)/(BRANCH)/(FILEPATH), but we want to be able to remove 131 # that (PROJECT) prefix from them. To do this without requiring the 132 # user to tell us how svnurl is split into ROOT and PROJECT, we do an 133 # 'svn info --xml' command at startup. This command will include a 134 # <root> element that tells us ROOT. We then strip this prefix from 135 # self.svnurl to determine PROJECT, and then later we strip the 136 # PROJECT prefix from the filenames reported by 'svn log --xml' to 137 # get a (BRANCH)/(FILEPATH) that can be passed to split_file() to 138 # turn into separate BRANCH and FILEPATH values. 139 140 # whew. 141 142 if self.project: 143 log.msg("SVNPoller: polling " + self.project) 144 else: 145 log.msg("SVNPoller: polling") 146 147 d = defer.succeed(None) 148 if not self._prefix: 149 d.addCallback(lambda _ : self.get_prefix()) 150 def set_prefix(prefix): 151 self._prefix = prefix

152 d.addCallback(set_prefix) 153 154 d.addCallback(self.get_logs) 155 d.addCallback(self.parse_logs) 156 d.addCallback(self.get_new_logentries) 157 d.addCallback(self.create_changes) 158 d.addCallback(self.submit_changes) 159 d.addCallback(self.finished_ok) 160 d.addErrback(log.err, 'SVNPoller: Error in while polling') # eat errors 161 return d

162

163 - def getProcessOutput(self, args):

164 # this exists so we can override it during the unit tests 165 d = utils.getProcessOutput(self.svnbin, args, self.environ) 166 return d

167

168 - def get_prefix(self):

169 args = ["info", "--xml", "--non-interactive", self.svnurl] 170 if self.svnuser: 171 args.extend(["--username=%s" % self.svnuser]) 172 if self.svnpasswd: 173 args.extend(["--password=%s" % self.svnpasswd]) 174 d = self.getProcessOutput(args) 175 def determine_prefix(output): 176 try: 177 doc = xml.dom.minidom.parseString(output) 178 except xml.parsers.expat.ExpatError: 179 log.msg("SVNPoller: SVNPoller._determine_prefix_2: ExpatError in '%s'" 180 % output) 181 raise 182 rootnodes = doc.getElementsByTagName("root") 183 if not rootnodes: 184 # this happens if the URL we gave was already the root. In this 185 # case, our prefix is empty. 186 self._prefix = "" 187 return self._prefix 188 rootnode = rootnodes[0] 189 root = "".join([c.data for c in rootnode.childNodes]) 190 # root will be a unicode string 191 assert self.svnurl.startswith(root), \ 192 ("svnurl='%s' doesn't start with <root>='%s'" % 193 (self.svnurl, root)) 194 prefix = self.svnurl[len(root):] 195 if prefix.startswith("/"): 196 prefix = prefix[1:] 197 log.msg("SVNPoller: svnurl=%s, root=%s, so prefix=%s" % 198 (self.svnurl, root, prefix)) 199 return prefix

200 d.addCallback(determine_prefix) 201 return d 202

203 - def get_logs(self, _):

204 args = [] 205 args.extend(["log", "--xml", "--verbose", "--non-interactive"]) 206 if self.svnuser: 207 args.extend(["--username=%s" % self.svnuser]) 208 if self.svnpasswd: 209 args.extend(["--password=%s" % self.svnpasswd]) 210 args.extend(["--limit=%d" % (self.histmax), self.svnurl]) 211 d = self.getProcessOutput(args) 212 return d

213

214 - def parse_logs(self, output):

215 # parse the XML output, return a list of <logentry> nodes 216 try: 217 doc = xml.dom.minidom.parseString(output) 218 except xml.parsers.expat.ExpatError: 219 log.msg("SVNPoller: SVNPoller.parse_logs: ExpatError in '%s'" % output) 220 raise 221 logentries = doc.getElementsByTagName("logentry") 222 return logentries

223 224

225 - def get_new_logentries(self, logentries):

226 last_change = old_last_change = self.last_change 227 228 # given a list of logentries, calculate new_last_change, and 229 # new_logentries, where new_logentries contains only the ones after 230 # last_change 231 232 new_last_change = None 233 new_logentries = [] 234 if logentries: 235 new_last_change = int(logentries[0].getAttribute("revision")) 236 237 if last_change is None: 238 # if this is the first time we've been run, ignore any changes 239 # that occurred before now. This prevents a build at every 240 # startup. 241 log.msg('SVNPoller: starting at change %s' % new_last_change) 242 elif last_change == new_last_change: 243 # an unmodified repository will hit this case 244 log.msg('SVNPoller: no changes') 245 else: 246 for el in logentries: 247 if last_change == int(el.getAttribute("revision")): 248 break 249 new_logentries.append(el) 250 new_logentries.reverse() # return oldest first 251 252 self.last_change = new_last_change 253 log.msg('SVNPoller: _process_changes %s .. %s' % 254 (old_last_change, new_last_change)) 255 return new_logentries

256 257

258 - def _get_text(self, element, tag_name):

259 try: 260 child_nodes = element.getElementsByTagName(tag_name)[0].childNodes 261 text = "".join([t.data for t in child_nodes]) 262 except: 263 text = "<unknown>" 264 return text

265

266 - def _transform_path(self, path):

267 assert path.startswith(self._prefix), \ 268 ("filepath '%s' should start with prefix '%s'" % 269 (path, self._prefix)) 270 relative_path = path[len(self._prefix):] 271 if relative_path.startswith("/"): 272 relative_path = relative_path[1:] 273 where = self.split_file(relative_path) 274 # 'where' is either None or (branch, final_path) 275 return where

276

277 - def create_changes(self, new_logentries):

278 changes = [] 279 280 for el in new_logentries: 281 revision = str(el.getAttribute("revision")) 282 283 revlink='' 284 285 if self.revlinktmpl: 286 if revision: 287 revlink = self.revlinktmpl % urllib.quote_plus(revision) 288 289 log.msg("Adding change revision %s" % (revision,)) 290 author = self._get_text(el, "author") 291 comments = self._get_text(el, "msg") 292 # there is a "date" field, but it provides localtime in the 293 # repository's timezone, whereas we care about buildmaster's 294 # localtime (since this will get used to position the boxes on 295 # the Waterfall display, etc). So ignore the date field, and 296 # addChange will fill in with the current time 297 branches = {} 298 try: 299 pathlist = el.getElementsByTagName("paths")[0] 300 except IndexError: # weird, we got an empty revision 301 log.msg("ignoring commit with no paths") 302 continue 303 304 for p in pathlist.getElementsByTagName("path"): 305 action = p.getAttribute("action") 306 path = "".join([t.data for t in p.childNodes]) 307 # the rest of buildbot is certaily not yet ready to handle 308 # unicode filenames, because they get put in RemoteCommands 309 # which get sent via PB to the buildslave, and PB doesn't 310 # handle unicode. 311 path = path.encode("ascii") 312 if path.startswith("/"): 313 path = path[1:] 314 where = self._transform_path(path) 315 316 # if 'where' is None, the file was outside any project that 317 # we care about and we should ignore it 318 if where: 319 branch, filename = where 320 if not branch in branches: 321 branches[branch] = { 'files': []} 322 branches[branch]['files'].append(filename) 323 324 if not branches[branch].has_key('action'): 325 branches[branch]['action'] = action 326 327 for branch in branches.keys(): 328 action = branches[branch]['action'] 329 files = branches[branch]['files'] 330 number_of_files_changed = len(files) 331 332 if action == u'D' and number_of_files_changed == 1 and files[0] == '': 333 log.msg("Ignoring deletion of branch '%s'" % branch) 334 else: 335 chdict = dict( 336 author=author, 337 files=files, 338 comments=comments, 339 revision=revision, 340 branch=branch, 341 revlink=revlink, 342 category=self.category, 343 repository=self.svnurl, 344 project = self.project) 345 changes.append(chdict) 346 347 return changes

348 349 @defer.deferredGenerator

350 - def submit_changes(self, changes):

351 for chdict in changes: 352 wfd = defer.waitForDeferred(self.master.addChange(src='svn', 353 **chdict)) 354 yield wfd 355 wfd.getResult()

356

357 - def finished_ok(self, res):

358 if self.cachepath: 359 f = open(self.cachepath, "w") 360 f.write(str(self.last_change)) 361 f.close() 362 363 log.msg("SVNPoller: finished polling %s" % res) 364 return res

365

Source Code for Module buildbot.changes.svnpoller