buildbot.changes.svnpoller

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 from __future__ import with_statement 17 18 19 # Based on the work of Dave Peticolas for the P4poll 20 # Changed to svn (using xml.dom.minidom) by Niklaus Giger 21 # Hacked beyond recognition by Brian Warner 22 23 from twisted.python import log 24 from twisted.internet import defer, utils 25 26 from buildbot import util 27 from buildbot.changes import base 28 29 import xml.dom.minidom 30 import os, urllib

31 32 # these split_file_* functions are available for use as values to the 33 # split_file= argument. 34 -def split_file_alwaystrunk(path):

35 return dict(path=path)

36

37 -def split_file_branches(path):

38 # turn "trunk/subdir/file.c" into (None, "subdir/file.c") 39 # and "trunk/subdir/" into (None, "subdir/") 40 # and "trunk/" into (None, "") 41 # and "branches/1.5.x/subdir/file.c" into ("branches/1.5.x", "subdir/file.c") 42 # and "branches/1.5.x/subdir/" into ("branches/1.5.x", "subdir/") 43 # and "branches/1.5.x/" into ("branches/1.5.x", "") 44 pieces = path.split('/') 45 if len(pieces) > 1 and pieces[0] == 'trunk': 46 return (None, '/'.join(pieces[1:])) 47 elif len(pieces) > 2 and pieces[0] == 'branches': 48 return ('/'.join(pieces[0:2]), '/'.join(pieces[2:])) 49 else: 50 return None

51

52 -def split_file_projects_branches(path):

53 # turn projectname/trunk/subdir/file.c into dict(project=projectname, branch=trunk, path=subdir/file.c) 54 if not "/" in path: 55 return None 56 project, path = path.split("/", 1) 57 f = split_file_branches(path) 58 if f: 59 info = dict(project=project, path=f[1]) 60 if f[0]: 61 info['branch'] = f[0] 62 return info 63 return f

64

65 -class SVNPoller(base.PollingChangeSource, util.ComparableMixin):

66 """ 67 Poll a Subversion repository for changes and submit them to the change 68 master. 69 """ 70 71 compare_attrs = ["svnurl", "split_file", 72 "svnuser", "svnpasswd", "project", 73 "pollInterval", "histmax", 74 "svnbin", "category", "cachepath"] 75 76 parent = None # filled in when we're added 77 last_change = None 78 loop = None 79

80 - def __init__(self, svnurl, split_file=None, 81 svnuser=None, svnpasswd=None, 82 pollInterval=10*60, histmax=100, 83 svnbin='svn', revlinktmpl='', category=None, 84 project='', cachepath=None, pollinterval=-2, 85 extra_args=None):

86 87 # for backward compatibility; the parameter used to be spelled with 'i' 88 if pollinterval != -2: 89 pollInterval = pollinterval 90 91 base.PollingChangeSource.__init__(self, name=svnurl, pollInterval=pollInterval) 92 93 if svnurl.endswith("/"): 94 svnurl = svnurl[:-1] # strip the trailing slash 95 self.svnurl = svnurl 96 self.extra_args = extra_args 97 self.split_file = split_file or split_file_alwaystrunk 98 self.svnuser = svnuser 99 self.svnpasswd = svnpasswd 100 101 self.revlinktmpl = revlinktmpl 102 103 self.environ = os.environ.copy() # include environment variables 104 # required for ssh-agent auth 105 106 self.svnbin = svnbin 107 self.histmax = histmax 108 self._prefix = None 109 self.category = category 110 self.project = project 111 112 self.cachepath = cachepath 113 if self.cachepath and os.path.exists(self.cachepath): 114 try: 115 with open(self.cachepath, "r") as f: 116 self.last_change = int(f.read().strip()) 117 log.msg("SVNPoller: SVNPoller(%s) setting last_change to %s" % (self.svnurl, self.last_change)) 118 # try writing it, too 119 with open(self.cachepath, "w") as f: 120 f.write(str(self.last_change)) 121 except: 122 self.cachepath = None 123 log.msg(("SVNPoller: SVNPoller(%s) cache file corrupt or unwriteable; " + 124 "skipping and not using") % self.svnurl) 125 log.err()

126

127 - def describe(self):

128 return "SVNPoller: watching %s" % self.svnurl

129

130 - def poll(self):

131 # Our return value is only used for unit testing. 132 133 # we need to figure out the repository root, so we can figure out 134 # repository-relative pathnames later. Each SVNURL is in the form 135 # (ROOT)/(PROJECT)/(BRANCH)/(FILEPATH), where (ROOT) is something 136 # like svn://svn.twistedmatrix.com/svn/Twisted (i.e. there is a 137 # physical repository at /svn/Twisted on that host), (PROJECT) is 138 # something like Projects/Twisted (i.e. within the repository's 139 # internal namespace, everything under Projects/Twisted/ has 140 # something to do with Twisted, but these directory names do not 141 # actually appear on the repository host), (BRANCH) is something like 142 # "trunk" or "branches/2.0.x", and (FILEPATH) is a tree-relative 143 # filename like "twisted/internet/defer.py". 144 145 # our self.svnurl attribute contains (ROOT)/(PROJECT) combined 146 # together in a way that we can't separate without svn's help. If the 147 # user is not using the split_file= argument, then self.svnurl might 148 # be (ROOT)/(PROJECT)/(BRANCH) . In any case, the filenames we will 149 # get back from 'svn log' will be of the form 150 # (PROJECT)/(BRANCH)/(FILEPATH), but we want to be able to remove 151 # that (PROJECT) prefix from them. To do this without requiring the 152 # user to tell us how svnurl is split into ROOT and PROJECT, we do an 153 # 'svn info --xml' command at startup. This command will include a 154 # <root> element that tells us ROOT. We then strip this prefix from 155 # self.svnurl to determine PROJECT, and then later we strip the 156 # PROJECT prefix from the filenames reported by 'svn log --xml' to 157 # get a (BRANCH)/(FILEPATH) that can be passed to split_file() to 158 # turn into separate BRANCH and FILEPATH values. 159 160 # whew. 161 162 if self.project: 163 log.msg("SVNPoller: polling " + self.project) 164 else: 165 log.msg("SVNPoller: polling") 166 167 d = defer.succeed(None) 168 if not self._prefix: 169 d.addCallback(lambda _ : self.get_prefix()) 170 def set_prefix(prefix): 171 self._prefix = prefix

172 d.addCallback(set_prefix) 173 174 d.addCallback(self.get_logs) 175 d.addCallback(self.parse_logs) 176 d.addCallback(self.get_new_logentries) 177 d.addCallback(self.create_changes) 178 d.addCallback(self.submit_changes) 179 d.addCallback(self.finished_ok) 180 d.addErrback(log.err, 'SVNPoller: Error in while polling') # eat errors 181 return d

182

183 - def getProcessOutput(self, args):

184 # this exists so we can override it during the unit tests 185 d = utils.getProcessOutput(self.svnbin, args, self.environ) 186 return d

187

188 - def get_prefix(self):

189 args = ["info", "--xml", "--non-interactive", self.svnurl] 190 if self.svnuser: 191 args.extend(["--username=%s" % self.svnuser]) 192 if self.svnpasswd: 193 args.extend(["--password=%s" % self.svnpasswd]) 194 if self.extra_args: 195 args.extend(self.extra_args) 196 d = self.getProcessOutput(args) 197 def determine_prefix(output): 198 try: 199 doc = xml.dom.minidom.parseString(output) 200 except xml.parsers.expat.ExpatError: 201 log.msg("SVNPoller: SVNPoller._determine_prefix_2: ExpatError in '%s'" 202 % output) 203 raise 204 rootnodes = doc.getElementsByTagName("root") 205 if not rootnodes: 206 # this happens if the URL we gave was already the root. In this 207 # case, our prefix is empty. 208 self._prefix = "" 209 return self._prefix 210 rootnode = rootnodes[0] 211 root = "".join([c.data for c in rootnode.childNodes]) 212 # root will be a unicode string 213 assert self.svnurl.startswith(root), \ 214 ("svnurl='%s' doesn't start with <root>='%s'" % 215 (self.svnurl, root)) 216 prefix = self.svnurl[len(root):] 217 if prefix.startswith("/"): 218 prefix = prefix[1:] 219 log.msg("SVNPoller: svnurl=%s, root=%s, so prefix=%s" % 220 (self.svnurl, root, prefix)) 221 return prefix

222 d.addCallback(determine_prefix) 223 return d 224

225 - def get_logs(self, _):

226 args = [] 227 args.extend(["log", "--xml", "--verbose", "--non-interactive"]) 228 if self.svnuser: 229 args.extend(["--username=%s" % self.svnuser]) 230 if self.svnpasswd: 231 args.extend(["--password=%s" % self.svnpasswd]) 232 if self.extra_args: 233 args.extend(self.extra_args) 234 args.extend(["--limit=%d" % (self.histmax), self.svnurl]) 235 d = self.getProcessOutput(args) 236 return d

237

238 - def parse_logs(self, output):

239 # parse the XML output, return a list of <logentry> nodes 240 try: 241 doc = xml.dom.minidom.parseString(output) 242 except xml.parsers.expat.ExpatError: 243 log.msg("SVNPoller: SVNPoller.parse_logs: ExpatError in '%s'" % output) 244 raise 245 logentries = doc.getElementsByTagName("logentry") 246 return logentries

247 248

249 - def get_new_logentries(self, logentries):

250 last_change = old_last_change = self.last_change 251 252 # given a list of logentries, calculate new_last_change, and 253 # new_logentries, where new_logentries contains only the ones after 254 # last_change 255 256 new_last_change = None 257 new_logentries = [] 258 if logentries: 259 new_last_change = int(logentries[0].getAttribute("revision")) 260 261 if last_change is None: 262 # if this is the first time we've been run, ignore any changes 263 # that occurred before now. This prevents a build at every 264 # startup. 265 log.msg('SVNPoller: starting at change %s' % new_last_change) 266 elif last_change == new_last_change: 267 # an unmodified repository will hit this case 268 log.msg('SVNPoller: no changes') 269 else: 270 for el in logentries: 271 if last_change == int(el.getAttribute("revision")): 272 break 273 new_logentries.append(el) 274 new_logentries.reverse() # return oldest first 275 276 self.last_change = new_last_change 277 log.msg('SVNPoller: _process_changes %s .. %s' % 278 (old_last_change, new_last_change)) 279 return new_logentries

280 281

282 - def _get_text(self, element, tag_name):

283 try: 284 child_nodes = element.getElementsByTagName(tag_name)[0].childNodes 285 text = "".join([t.data for t in child_nodes]) 286 except: 287 text = "<unknown>" 288 return text

289

290 - def _transform_path(self, path):

291 assert path.startswith(self._prefix), \ 292 ("filepath '%s' should start with prefix '%s'" % 293 (path, self._prefix)) 294 relative_path = path[len(self._prefix):] 295 if relative_path.startswith("/"): 296 relative_path = relative_path[1:] 297 where = self.split_file(relative_path) 298 # 'where' is either None, (branch, final_path) or a dict 299 if not where: 300 return 301 if isinstance(where, tuple): 302 where = dict(branch=where[0], path=where[1]) 303 return where

304

305 - def create_changes(self, new_logentries):

306 changes = [] 307 308 for el in new_logentries: 309 revision = str(el.getAttribute("revision")) 310 311 revlink='' 312 313 if self.revlinktmpl: 314 if revision: 315 revlink = self.revlinktmpl % urllib.quote_plus(revision) 316 317 log.msg("Adding change revision %s" % (revision,)) 318 author = self._get_text(el, "author") 319 comments = self._get_text(el, "msg") 320 # there is a "date" field, but it provides localtime in the 321 # repository's timezone, whereas we care about buildmaster's 322 # localtime (since this will get used to position the boxes on 323 # the Waterfall display, etc). So ignore the date field, and 324 # addChange will fill in with the current time 325 branches = {} 326 try: 327 pathlist = el.getElementsByTagName("paths")[0] 328 except IndexError: # weird, we got an empty revision 329 log.msg("ignoring commit with no paths") 330 continue 331 332 for p in pathlist.getElementsByTagName("path"): 333 kind = p.getAttribute("kind") 334 action = p.getAttribute("action") 335 path = "".join([t.data for t in p.childNodes]) 336 # the rest of buildbot is certaily not yet ready to handle 337 # unicode filenames, because they get put in RemoteCommands 338 # which get sent via PB to the buildslave, and PB doesn't 339 # handle unicode. 340 path = path.encode("ascii") 341 if path.startswith("/"): 342 path = path[1:] 343 if kind == "dir" and not path.endswith("/"): 344 path += "/" 345 where = self._transform_path(path) 346 347 # if 'where' is None, the file was outside any project that 348 # we care about and we should ignore it 349 if where: 350 branch = where.get("branch", None) 351 filename = where["path"] 352 if not branch in branches: 353 branches[branch] = { 'files': [], 'number_of_directories': 0} 354 if filename == "": 355 # root directory of branch 356 branches[branch]['files'].append(filename) 357 branches[branch]['number_of_directories'] += 1 358 elif filename.endswith("/"): 359 # subdirectory of branch 360 branches[branch]['files'].append(filename[:-1]) 361 branches[branch]['number_of_directories'] += 1 362 else: 363 branches[branch]['files'].append(filename) 364 365 if not branches[branch].has_key('action'): 366 branches[branch]['action'] = action 367 368 for key in ("repository", "project", "codebase"): 369 if key in where: 370 branches[branch][key] = where[key] 371 372 for branch in branches.keys(): 373 action = branches[branch]['action'] 374 files = branches[branch]['files'] 375 376 number_of_directories_changed = branches[branch]['number_of_directories'] 377 number_of_files_changed = len(files) 378 379 if action == u'D' and number_of_directories_changed == 1 and number_of_files_changed == 1 and files[0] == '': 380 log.msg("Ignoring deletion of branch '%s'" % branch) 381 else: 382 chdict = dict( 383 author=author, 384 files=files, 385 comments=comments, 386 revision=revision, 387 branch=branch, 388 revlink=revlink, 389 category=self.category, 390 repository=branches[branch].get('repository', self.svnurl), 391 project=branches[branch].get('project', self.project), 392 codebase=branches[branch].get('codebase', None)) 393 changes.append(chdict) 394 395 return changes

396 397 @defer.inlineCallbacks

398 - def submit_changes(self, changes):

399 for chdict in changes: 400 yield self.master.addChange(src='svn', **chdict)

401

402 - def finished_ok(self, res):

403 if self.cachepath: 404 with open(self.cachepath, "w") as f: 405 f.write(str(self.last_change)) 406 407 log.msg("SVNPoller: finished polling %s" % res) 408 return res

409

Source Code for Module buildbot.changes.svnpoller