buildbot.changes.svnpoller

41 """This source will poll a Subversion repository for changes and submit 42 them to the change master.""" 43 44 compare_attrs = ["svnurl", "split_file_function", 45 "svnuser", "svnpasswd", 46 "pollinterval", "histmax", 47 "svnbin", "category"] 48 49 parent = None # filled in when we're added 50 last_change = None 51 loop = None 52 working = False 53

54 - def __init__(self, svnurl, split_file=None, 55 svnuser=None, svnpasswd=None, 56 pollinterval=10*60, histmax=100, 57 svnbin='svn', revlinktmpl='', category=None):

58 """ 59 @type svnurl: string 60 @param svnurl: the SVN URL that describes the repository and 61 subdirectory to watch. If this ChangeSource should 62 only pay attention to a single branch, this should 63 point at the repository for that branch, like 64 svn://svn.twistedmatrix.com/svn/Twisted/trunk . If it 65 should follow multiple branches, point it at the 66 repository directory that contains all the branches 67 like svn://svn.twistedmatrix.com/svn/Twisted and also 68 provide a branch-determining function. 69 70 Each file in the repository has a SVN URL in the form 71 (SVNURL)/(BRANCH)/(FILEPATH), where (BRANCH) could be 72 empty or not, depending upon your branch-determining 73 function. Only files that start with (SVNURL)/(BRANCH) 74 will be monitored. The Change objects that are sent to 75 the Schedulers will see (FILEPATH) for each modified 76 file. 77 78 @type split_file: callable or None 79 @param split_file: a function that is called with a string of the 80 form (BRANCH)/(FILEPATH) and should return a tuple 81 (BRANCH, FILEPATH). This function should match 82 your repository's branch-naming policy. Each 83 changed file has a fully-qualified URL that can be 84 split into a prefix (which equals the value of the 85 'svnurl' argument) and a suffix; it is this suffix 86 which is passed to the split_file function. 87 88 If the function returns None, the file is ignored. 89 Use this to indicate that the file is not a part 90 of this project. 91 92 For example, if your repository puts the trunk in 93 trunk/... and branches are in places like 94 branches/1.5/..., your split_file function could 95 look like the following (this function is 96 available as svnpoller.split_file_branches):: 97 98 pieces = path.split('/') 99 if pieces[0] == 'trunk': 100 return (None, '/'.join(pieces[1:])) 101 elif pieces[0] == 'branches': 102 return ('/'.join(pieces[0:2]), 103 '/'.join(pieces[2:])) 104 else: 105 return None 106 107 If instead your repository layout puts the trunk 108 for ProjectA in trunk/ProjectA/... and the 1.5 109 branch in branches/1.5/ProjectA/..., your 110 split_file function could look like:: 111 112 pieces = path.split('/') 113 if pieces[0] == 'trunk': 114 branch = None 115 pieces.pop(0) # remove 'trunk' 116 elif pieces[0] == 'branches': 117 pieces.pop(0) # remove 'branches' 118 # grab branch name 119 branch = 'branches/' + pieces.pop(0) 120 else: 121 return None # something weird 122 projectname = pieces.pop(0) 123 if projectname != 'ProjectA': 124 return None # wrong project 125 return (branch, '/'.join(pieces)) 126 127 The default of split_file= is None, which 128 indicates that no splitting should be done. This 129 is equivalent to the following function:: 130 131 return (None, path) 132 133 If you wish, you can override the split_file 134 method with the same sort of function instead of 135 passing in a split_file= argument. 136 137 138 @type svnuser: string 139 @param svnuser: If set, the --username option will be added to 140 the 'svn log' command. You may need this to get 141 access to a private repository. 142 @type svnpasswd: string 143 @param svnpasswd: If set, the --password option will be added. 144 145 @type pollinterval: int 146 @param pollinterval: interval in seconds between polls. The default 147 is 600 seconds (10 minutes). Smaller values 148 decrease the latency between the time a change 149 is recorded and the time the buildbot notices 150 it, but it also increases the system load. 151 152 @type histmax: int 153 @param histmax: maximum number of changes to look back through. 154 The default is 100. Smaller values decrease 155 system load, but if more than histmax changes 156 are recorded between polls, the extra ones will 157 be silently lost. 158 159 @type svnbin: string 160 @param svnbin: path to svn binary, defaults to just 'svn'. Use 161 this if your subversion command lives in an 162 unusual location. 163 164 @type revlinktmpl: string 165 @param revlinktmpl: A format string to use for hyperlinks to revision 166 information. For example, setting this to 167 "http://reposerver/websvn/revision.php?rev=%s" 168 would create suitable links on the build pages 169 to information in websvn on each revision. 170 171 @type category: string 172 @param category: A single category associated with the changes that 173 could be used by schedulers watch for branches of a 174 certain name AND category. 175 """ 176 177 if svnurl.endswith("/"): 178 svnurl = svnurl[:-1] # strip the trailing slash 179 self.svnurl = svnurl 180 self.split_file_function = split_file or split_file_alwaystrunk 181 self.svnuser = svnuser 182 self.svnpasswd = svnpasswd 183 184 self.revlinktmpl = revlinktmpl 185 186 self.environ = os.environ.copy() # include environment variables 187 # required for ssh-agent auth 188 189 self.svnbin = svnbin 190 self.pollinterval = pollinterval 191 self.histmax = histmax 192 self._prefix = None 193 self.overrun_counter = 0 194 self.loop = LoopingCall(self.checksvn) 195 self.category = category

196

197 - def split_file(self, path):

198 # use getattr() to avoid turning this function into a bound method, 199 # which would require it to have an extra 'self' argument 200 f = getattr(self, "split_file_function") 201 return f(path)

202

203 - def startService(self):

204 log.msg("SVNPoller(%s) starting" % self.svnurl) 205 base.ChangeSource.startService(self) 206 # Don't start the loop just yet because the reactor isn't running. 207 # Give it a chance to go and install our SIGCHLD handler before 208 # spawning processes. 209 reactor.callLater(0, self.loop.start, self.pollinterval)

210

211 - def stopService(self):

212 log.msg("SVNPoller(%s) shutting down" % self.svnurl) 213 self.loop.stop() 214 return base.ChangeSource.stopService(self)

215

216 - def describe(self):

217 return "SVNPoller watching %s" % self.svnurl

218

219 - def checksvn(self):

220 # Our return value is only used for unit testing. 221 222 # we need to figure out the repository root, so we can figure out 223 # repository-relative pathnames later. Each SVNURL is in the form 224 # (ROOT)/(PROJECT)/(BRANCH)/(FILEPATH), where (ROOT) is something 225 # like svn://svn.twistedmatrix.com/svn/Twisted (i.e. there is a 226 # physical repository at /svn/Twisted on that host), (PROJECT) is 227 # something like Projects/Twisted (i.e. within the repository's 228 # internal namespace, everything under Projects/Twisted/ has 229 # something to do with Twisted, but these directory names do not 230 # actually appear on the repository host), (BRANCH) is something like 231 # "trunk" or "branches/2.0.x", and (FILEPATH) is a tree-relative 232 # filename like "twisted/internet/defer.py". 233 234 # our self.svnurl attribute contains (ROOT)/(PROJECT) combined 235 # together in a way that we can't separate without svn's help. If the 236 # user is not using the split_file= argument, then self.svnurl might 237 # be (ROOT)/(PROJECT)/(BRANCH) . In any case, the filenames we will 238 # get back from 'svn log' will be of the form 239 # (PROJECT)/(BRANCH)/(FILEPATH), but we want to be able to remove 240 # that (PROJECT) prefix from them. To do this without requiring the 241 # user to tell us how svnurl is split into ROOT and PROJECT, we do an 242 # 'svn info --xml' command at startup. This command will include a 243 # <root> element that tells us ROOT. We then strip this prefix from 244 # self.svnurl to determine PROJECT, and then later we strip the 245 # PROJECT prefix from the filenames reported by 'svn log --xml' to 246 # get a (BRANCH)/(FILEPATH) that can be passed to split_file() to 247 # turn into separate BRANCH and FILEPATH values. 248 249 # whew. 250 251 if self.working: 252 log.msg("SVNPoller(%s) overrun: timer fired but the previous " 253 "poll had not yet finished." % self.svnurl) 254 self.overrun_counter += 1 255 return defer.succeed(None) 256 self.working = True 257 258 log.msg("SVNPoller polling") 259 if not self._prefix: 260 # this sets self._prefix when it finishes. It fires with 261 # self._prefix as well, because that makes the unit tests easier 262 # to write. 263 d = self.get_root() 264 d.addCallback(self.determine_prefix) 265 else: 266 d = defer.succeed(self._prefix) 267 268 d.addCallback(self.get_logs) 269 d.addCallback(self.parse_logs) 270 d.addCallback(self.get_new_logentries) 271 d.addCallback(self.create_changes) 272 d.addCallback(self.submit_changes) 273 d.addCallbacks(self.finished_ok, self.finished_failure) 274 return d

275

276 - def getProcessOutput(self, args):

277 # this exists so we can override it during the unit tests 278 d = utils.getProcessOutput(self.svnbin, args, self.environ) 279 return d

280

281 - def get_root(self):

282 args = ["info", "--xml", "--non-interactive", self.svnurl] 283 if self.svnuser: 284 args.extend(["--username=%s" % self.svnuser]) 285 if self.svnpasswd: 286 args.extend(["--password=%s" % self.svnpasswd]) 287 d = self.getProcessOutput(args) 288 return d

289

290 - def determine_prefix(self, output):

291 try: 292 doc = xml.dom.minidom.parseString(output) 293 except xml.parsers.expat.ExpatError: 294 log.msg("SVNPoller._determine_prefix_2: ExpatError in '%s'" 295 % output) 296 raise 297 rootnodes = doc.getElementsByTagName("root") 298 if not rootnodes: 299 # this happens if the URL we gave was already the root. In this 300 # case, our prefix is empty. 301 self._prefix = "" 302 return self._prefix 303 rootnode = rootnodes[0] 304 root = "".join([c.data for c in rootnode.childNodes]) 305 # root will be a unicode string 306 _assert(self.svnurl.startswith(root), 307 "svnurl='%s' doesn't start with <root>='%s'" % 308 (self.svnurl, root)) 309 self._prefix = self.svnurl[len(root):] 310 if self._prefix.startswith("/"): 311 self._prefix = self._prefix[1:] 312 log.msg("SVNPoller: svnurl=%s, root=%s, so prefix=%s" % 313 (self.svnurl, root, self._prefix)) 314 return self._prefix

315

316 - def get_logs(self, ignored_prefix=None):

317 args = [] 318 args.extend(["log", "--xml", "--verbose", "--non-interactive"]) 319 if self.svnuser: 320 args.extend(["--username=%s" % self.svnuser]) 321 if self.svnpasswd: 322 args.extend(["--password=%s" % self.svnpasswd]) 323 args.extend(["--limit=%d" % (self.histmax), self.svnurl]) 324 d = self.getProcessOutput(args) 325 return d

326

327 - def parse_logs(self, output):

328 # parse the XML output, return a list of <logentry> nodes 329 try: 330 doc = xml.dom.minidom.parseString(output) 331 except xml.parsers.expat.ExpatError: 332 log.msg("SVNPoller.parse_logs: ExpatError in '%s'" % output) 333 raise 334 logentries = doc.getElementsByTagName("logentry") 335 return logentries

336 337

338 - def _filter_new_logentries(self, logentries, last_change):

339 # given a list of logentries, return a tuple of (new_last_change, 340 # new_logentries), where new_logentries contains only the ones after 341 # last_change 342 if not logentries: 343 # no entries, so last_change must stay at None 344 return (None, []) 345 346 mostRecent = int(logentries[0].getAttribute("revision")) 347 348 if last_change is None: 349 # if this is the first time we've been run, ignore any changes 350 # that occurred before now. This prevents a build at every 351 # startup. 352 log.msg('svnPoller: starting at change %s' % mostRecent) 353 return (mostRecent, []) 354 355 if last_change == mostRecent: 356 # an unmodified repository will hit this case 357 log.msg('svnPoller: _process_changes last %s mostRecent %s' % ( 358 last_change, mostRecent)) 359 return (mostRecent, []) 360 361 new_logentries = [] 362 for el in logentries: 363 if last_change == int(el.getAttribute("revision")): 364 break 365 new_logentries.append(el) 366 new_logentries.reverse() # return oldest first 367 return (mostRecent, new_logentries)

368

369 - def get_new_logentries(self, logentries):

370 last_change = self.last_change 371 (new_last_change, 372 new_logentries) = self._filter_new_logentries(logentries, 373 self.last_change) 374 self.last_change = new_last_change 375 log.msg('svnPoller: _process_changes %s .. %s' % 376 (last_change, new_last_change)) 377 return new_logentries

378 379

380 - def _get_text(self, element, tag_name):

381 try: 382 child_nodes = element.getElementsByTagName(tag_name)[0].childNodes 383 text = "".join([t.data for t in child_nodes]) 384 except: 385 text = "<unknown>" 386 return text

387

388 - def _transform_path(self, path):

389 _assert(path.startswith(self._prefix), 390 "filepath '%s' should start with prefix '%s'" % 391 (path, self._prefix)) 392 relative_path = path[len(self._prefix):] 393 if relative_path.startswith("/"): 394 relative_path = relative_path[1:] 395 where = self.split_file(relative_path) 396 # 'where' is either None or (branch, final_path) 397 return where

398

399 - def create_changes(self, new_logentries):

400 changes = [] 401 402 for el in new_logentries: 403 revision = str(el.getAttribute("revision")) 404 405 revlink='' 406 407 if self.revlinktmpl: 408 if revision: 409 revlink = self.revlinktmpl % urllib.quote_plus(revision) 410 411 log.msg("Adding change revision %s" % (revision,)) 412 # TODO: the rest of buildbot may not be ready for unicode 'who' 413 # values 414 author = self._get_text(el, "author") 415 comments = self._get_text(el, "msg") 416 # there is a "date" field, but it provides localtime in the 417 # repository's timezone, whereas we care about buildmaster's 418 # localtime (since this will get used to position the boxes on 419 # the Waterfall display, etc). So ignore the date field and use 420 # our local clock instead. 421 #when = self._get_text(el, "date") 422 #when = time.mktime(time.strptime("%.19s" % when, 423 # "%Y-%m-%dT%H:%M:%S")) 424 branches = {} 425 pathlist = el.getElementsByTagName("paths")[0] 426 for p in pathlist.getElementsByTagName("path"): 427 action = p.getAttribute("action") 428 path = "".join([t.data for t in p.childNodes]) 429 # the rest of buildbot is certaily not yet ready to handle 430 # unicode filenames, because they get put in RemoteCommands 431 # which get sent via PB to the buildslave, and PB doesn't 432 # handle unicode. 433 path = path.encode("ascii") 434 if path.startswith("/"): 435 path = path[1:] 436 where = self._transform_path(path) 437 438 # if 'where' is None, the file was outside any project that 439 # we care about and we should ignore it 440 if where: 441 branch, filename = where 442 if not branch in branches: 443 branches[branch] = { 'files': []} 444 branches[branch]['files'].append(filename) 445 446 if not branches[branch].has_key('action'): 447 branches[branch]['action'] = action 448 449 for branch in branches.keys(): 450 action = branches[branch]['action'] 451 files = branches[branch]['files'] 452 number_of_files_changed = len(files) 453 454 if action == u'D' and number_of_files_changed == 1 and files[0] == '': 455 log.msg("Ignoring deletion of branch '%s'" % branch) 456 else: 457 c = Change(who=author, 458 files=files, 459 comments=comments, 460 revision=revision, 461 branch=branch, 462 revlink=revlink, 463 category=self.category, 464 repository=self.svnurl) 465 changes.append(c) 466 467 return changes

468

469 - def submit_changes(self, changes):

470 for c in changes: 471 self.parent.addChange(c)

472

473 - def finished_ok(self, res):

474 log.msg("SVNPoller finished polling %s" % res) 475 assert self.working 476 self.working = False 477 return res

478

479 - def finished_failure(self, f):

480 log.msg("SVNPoller failed %s" % f) 481 assert self.working 482 self.working = False 483 return None # eat the failure

Source Code for Module buildbot.changes.svnpoller