Package buildbot :: Package changes :: Module mail
[frames] | no frames]

Source Code for Module buildbot.changes.mail

  1  # This file is part of Buildbot.  Buildbot is free software: you can 
  2  # redistribute it and/or modify it under the terms of the GNU General Public 
  3  # License as published by the Free Software Foundation, version 2. 
  4  # 
  5  # This program is distributed in the hope that it will be useful, but WITHOUT 
  6  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
  7  # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more 
  8  # details. 
  9  # 
 10  # You should have received a copy of the GNU General Public License along with 
 11  # this program; if not, write to the Free Software Foundation, Inc., 51 
 12  # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 
 13  # 
 14  # Copyright Buildbot Team Members 
 15   
 16   
 17  """ 
 18  Parse various kinds of 'CVS notify' email. 
 19  """ 
 20  import os, re 
 21  import time, calendar 
 22  import datetime 
 23  from email import message_from_file 
 24  from email.Utils import parseaddr, parsedate_tz, mktime_tz 
 25  from email.Iterators import body_line_iterator 
 26   
 27  from zope.interface import implements 
 28  from twisted.python import log 
 29  from buildbot import util 
 30  from buildbot.interfaces import IChangeSource 
 31  from buildbot.changes import changes 
 32  from buildbot.changes.maildir import MaildirService 
 33   
34 -class MaildirSource(MaildirService, util.ComparableMixin):
35 """This source will watch a maildir that is subscribed to a FreshCVS 36 change-announcement mailing list. 37 """ 38 implements(IChangeSource) 39 40 compare_attrs = ["basedir", "pollinterval", "prefix"] 41 name = None 42
43 - def __init__(self, maildir, prefix=None, category='', repository=''):
44 MaildirService.__init__(self, maildir) 45 self.prefix = prefix 46 self.category = category 47 self.repository = repository 48 if prefix and not prefix.endswith("/"): 49 log.msg("%s: you probably want your prefix=('%s') to end with " 50 "a slash")
51
52 - def describe(self):
53 return "%s mailing list in maildir %s" % (self.name, self.basedir)
54
55 - def messageReceived(self, filename):
56 path = os.path.join(self.basedir, "new", filename) 57 change = self.parse_file(open(path, "r"), self.prefix) 58 if change: 59 self.parent.addChange(change) 60 os.rename(os.path.join(self.basedir, "new", filename), 61 os.path.join(self.basedir, "cur", filename))
62
63 - def parse_file(self, fd, prefix=None):
64 m = message_from_file(fd) 65 return self.parse(m, prefix)
66
67 -class CVSMaildirSource(MaildirSource):
68 name = "CVSMaildirSource" 69
70 - def __init__(self, maildir, prefix=None, category='', 71 repository='', urlmaker=None, properties={}):
72 """If urlmaker is defined, it will be called with three arguments: 73 filename, previous version, new version. It returns a url for that 74 file.""" 75 MaildirSource.__init__(self, maildir, prefix, category, repository) 76 self.urlmaker = urlmaker 77 self.properties = properties
78
79 - def parse(self, m, prefix=None):
80 """Parse messages sent by the 'buildbot-cvs-mail' program. 81 """ 82 # The mail is sent from the person doing the checkin. Assume that the 83 # local username is enough to identify them (this assumes a one-server 84 # cvs-over-rsh environment rather than the server-dirs-shared-over-NFS 85 # model) 86 name, addr = parseaddr(m["from"]) 87 if not addr: 88 return None # no From means this message isn't from buildbot-cvs-mail 89 at = addr.find("@") 90 if at == -1: 91 who = addr # might still be useful 92 else: 93 who = addr[:at] 94 95 # CVS accecpts RFC822 dates. buildbot-cvs-mail adds the date as 96 # part of the mail header, so use that. 97 # This assumes cvs is being access via ssh or pserver, so the time 98 # will be the CVS server's time. 99 100 # calculate a "revision" based on that timestamp, or the current time 101 # if we're unable to parse the date. 102 log.msg('Processing CVS mail') 103 dateTuple = parsedate_tz(m["date"]) 104 if dateTuple == None: 105 when = util.now() 106 else: 107 when = mktime_tz(dateTuple) 108 109 theTime = datetime.datetime.utcfromtimestamp(float(when)) 110 rev = theTime.strftime('%Y-%m-%d %H:%M:%S') 111 112 catRE = re.compile( '^Category:\s*(\S.*)') 113 cvsRE = re.compile( '^CVSROOT:\s*(\S.*)') 114 cvsmodeRE = re.compile( '^Cvsmode:\s*(\S.*)') 115 filesRE = re.compile( '^Files:\s*(\S.*)') 116 modRE = re.compile( '^Module:\s*(\S.*)') 117 pathRE = re.compile( '^Path:\s*(\S.*)') 118 projRE = re.compile( '^Project:\s*(\S.*)') 119 singleFileRE = re.compile( '(.*) (NONE|\d(\.|\d)+) (NONE|\d(\.|\d)+)') 120 tagRE = re.compile( '^\s+Tag:\s*(\S.*)') 121 updateRE = re.compile( '^Update of:\s*(\S.*)') 122 comments = "" 123 branch = None 124 cvsroot = None 125 fileList = None 126 files = [] 127 isdir = 0 128 path = None 129 project = None 130 131 lines = list(body_line_iterator(m)) 132 while lines: 133 line = lines.pop(0) 134 m = catRE.match(line) 135 if m: 136 category = m.group(1) 137 continue 138 m = cvsRE.match(line) 139 if m: 140 cvsroot = m.group(1) 141 continue 142 m = cvsmodeRE.match(line) 143 if m: 144 cvsmode = m.group(1) 145 continue 146 m = filesRE.match(line) 147 if m: 148 fileList = m.group(1) 149 continue 150 m = modRE.match(line) 151 if m: 152 # We don't actually use this 153 #module = m.group(1) 154 continue 155 m = pathRE.match(line) 156 if m: 157 path = m.group(1) 158 continue 159 m = projRE.match(line) 160 if m: 161 project = m.group(1) 162 continue 163 m = tagRE.match(line) 164 if m: 165 branch = m.group(1) 166 continue 167 m = updateRE.match(line) 168 if m: 169 # We don't actually use this 170 #updateof = m.group(1) 171 continue 172 if line == "Log Message:\n": 173 break 174 175 # CVS 1.11 lists files as: 176 # repo/path file,old-version,new-version file2,old-version,new-version 177 # Version 1.12 lists files as: 178 # file1 old-version new-version file2 old-version new-version 179 # 180 # files consists of tuples of 'file-name old-version new-version' 181 # The versions are either dotted-decimal version numbers, ie 1.1 182 # or NONE. New files are of the form 'NONE NUMBER', while removed 183 # files are 'NUMBER NONE'. 'NONE' is a literal string 184 # Parsing this instead of files list in 'Added File:' etc 185 # makes it possible to handle files with embedded spaces, though 186 # it could fail if the filename was 'bad 1.1 1.2' 187 # For cvs version 1.11, we expect 188 # my_module new_file.c,NONE,1.1 189 # my_module removed.txt,1.2,NONE 190 # my_module modified_file.c,1.1,1.2 191 # While cvs version 1.12 gives us 192 # new_file.c NONE 1.1 193 # removed.txt 1.2 NONE 194 # modified_file.c 1.1,1.2 195 196 if fileList is None: 197 log.msg('CVSMaildirSource Mail with no files. Ignoring') 198 return None # We don't have any files. Email not from CVS 199 200 if cvsmode == '1.11': 201 # Please, no repo paths with spaces! 202 m = re.search('([^ ]*) ', fileList) 203 if m: 204 path = m.group(1) 205 else: 206 log.msg('CVSMaildirSource can\'t get path from file list. Ignoring mail') 207 return 208 fileList = fileList[len(path):].strip() 209 singleFileRE = re.compile( '(.+?),(NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+)),(NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+))(?: |$)') 210 elif cvsmode == '1.12': 211 singleFileRE = re.compile( '(.+?) (NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+)) (NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+))(?: |$)') 212 if path is None: 213 raise ValueError('CVSMaildirSource cvs 1.12 require path. Check cvs loginfo config') 214 else: 215 raise ValueError('Expected cvsmode 1.11 or 1.12. got: %s' % cvsmode) 216 217 log.msg("CVSMaildirSource processing filelist: %s" % fileList) 218 links = [] 219 while(fileList): 220 m = singleFileRE.match(fileList) 221 if m: 222 curFile = path + '/' + m.group(1) 223 oldRev = m.group(2) 224 newRev = m.group(3) 225 files.append( curFile ) 226 if self.urlmaker: 227 links.append(self.urlmaker(curFile, oldRev, newRev )) 228 fileList = fileList[m.end():] 229 else: 230 log.msg('CVSMaildirSource no files matched regex. Ignoring') 231 return None # bail - we couldn't parse the files that changed 232 # Now get comments 233 while lines: 234 line = lines.pop(0) 235 comments += line 236 237 comments = comments.rstrip() + "\n" 238 if comments == '\n': 239 comments = None 240 change = changes.Change(who, files, comments, isdir, when=when, 241 branch=branch, revision=rev, 242 category=category, 243 repository=cvsroot, 244 project=project, 245 links=links, 246 properties=self.properties) 247 return change
248 249 # svn "commit-email.pl" handler. The format is very similar to freshcvs mail; 250 # here's a sample: 251 252 # From: username [at] apache.org [slightly obfuscated to avoid spam here] 253 # To: commits [at] spamassassin.apache.org 254 # Subject: svn commit: r105955 - in spamassassin/trunk: . lib/Mail 255 # ... 256 # 257 # Author: username 258 # Date: Sat Nov 20 00:17:49 2004 [note: TZ = local tz on server!] 259 # New Revision: 105955 260 # 261 # Modified: [also Removed: and Added:] 262 # [filename] 263 # ... 264 # Log: 265 # [log message] 266 # ... 267 # 268 # 269 # Modified: spamassassin/trunk/lib/Mail/SpamAssassin.pm 270 # [unified diff] 271 # 272 # [end of mail] 273
274 -class SVNCommitEmailMaildirSource(MaildirSource):
275 name = "SVN commit-email.pl" 276
277 - def parse(self, m, prefix=None):
278 """Parse messages sent by the svn 'commit-email.pl' trigger. 279 """ 280 281 # The mail is sent from the person doing the checkin. Assume that the 282 # local username is enough to identify them (this assumes a one-server 283 # cvs-over-rsh environment rather than the server-dirs-shared-over-NFS 284 # model) 285 name, addr = parseaddr(m["from"]) 286 if not addr: 287 return None # no From means this message isn't from FreshCVS 288 at = addr.find("@") 289 if at == -1: 290 who = addr # might still be useful 291 else: 292 who = addr[:at] 293 294 # we take the time of receipt as the time of checkin. Not correct (it 295 # depends upon the email latency), but it avoids the 296 # out-of-order-changes issue. Also syncmail doesn't give us anything 297 # better to work with, unless you count pulling the v1-vs-v2 298 # timestamp out of the diffs, which would be ugly. TODO: Pulling the 299 # 'Date:' header from the mail is a possibility, and 300 # email.Utils.parsedate_tz may be useful. It should be configurable, 301 # however, because there are a lot of broken clocks out there. 302 when = util.now() 303 304 files = [] 305 comments = "" 306 lines = list(body_line_iterator(m)) 307 rev = None 308 while lines: 309 line = lines.pop(0) 310 311 # "Author: jmason" 312 match = re.search(r"^Author: (\S+)", line) 313 if match: 314 who = match.group(1) 315 316 # "New Revision: 105955" 317 match = re.search(r"^New Revision: (\d+)", line) 318 if match: 319 rev = match.group(1) 320 321 # possible TODO: use "Date: ..." data here instead of time of 322 # commit message receipt, above. however, this timestamp is 323 # specified *without* a timezone, in the server's local TZ, so to 324 # be accurate buildbot would need a config setting to specify the 325 # source server's expected TZ setting! messy. 326 327 # this stanza ends with the "Log:" 328 if (line == "Log:\n"): 329 break 330 331 # commit message is terminated by the file-listing section 332 while lines: 333 line = lines.pop(0) 334 if (line == "Modified:\n" or 335 line == "Added:\n" or 336 line == "Removed:\n"): 337 break 338 comments += line 339 comments = comments.rstrip() + "\n" 340 341 while lines: 342 line = lines.pop(0) 343 if line == "\n": 344 break 345 if line.find("Modified:\n") == 0: 346 continue # ignore this line 347 if line.find("Added:\n") == 0: 348 continue # ignore this line 349 if line.find("Removed:\n") == 0: 350 continue # ignore this line 351 line = line.strip() 352 353 thesefiles = line.split(" ") 354 for f in thesefiles: 355 if prefix: 356 # insist that the file start with the prefix: we may get 357 # changes we don't care about too 358 if f.startswith(prefix): 359 f = f[len(prefix):] 360 else: 361 log.msg("ignored file from svn commit: prefix '%s' " 362 "does not match filename '%s'" % (prefix, f)) 363 continue 364 365 # TODO: figure out how new directories are described, set 366 # .isdir 367 files.append(f) 368 369 if not files: 370 log.msg("no matching files found, ignoring commit") 371 return None 372 373 return changes.Change(who, files, comments, when=when, revision=rev)
374 375 # bzr Launchpad branch subscription mails. Sample mail: 376 # 377 # From: noreply@launchpad.net 378 # Subject: [Branch ~knielsen/maria/tmp-buildbot-test] Rev 2701: test add file 379 # To: Joe <joe@acme.com> 380 # ... 381 # 382 # ------------------------------------------------------------ 383 # revno: 2701 384 # committer: Joe <joe@acme.com> 385 # branch nick: tmpbb 386 # timestamp: Fri 2009-05-15 10:35:43 +0200 387 # message: 388 # test add file 389 # added: 390 # test-add-file 391 # 392 # 393 # -- 394 # 395 # https://code.launchpad.net/~knielsen/maria/tmp-buildbot-test 396 # 397 # You are subscribed to branch lp:~knielsen/maria/tmp-buildbot-test. 398 # To unsubscribe from this branch go to https://code.launchpad.net/~knielsen/maria/tmp-buildbot-test/+edit-subscription. 399 # 400 # [end of mail] 401
402 -class BzrLaunchpadEmailMaildirSource(MaildirSource):
403 name = "Launchpad" 404 405 compare_attrs = MaildirSource.compare_attrs + ["branchMap", "defaultBranch"] 406
407 - def __init__(self, maildir, prefix=None, branchMap=None, defaultBranch=None, **kwargs):
408 self.branchMap = branchMap 409 self.defaultBranch = defaultBranch 410 MaildirSource.__init__(self, maildir, prefix, **kwargs)
411
412 - def parse(self, m, prefix=None):
413 """Parse branch notification messages sent by Launchpad. 414 """ 415 416 subject = m["subject"] 417 match = re.search(r"^\s*\[Branch\s+([^]]+)\]", subject) 418 if match: 419 repository = match.group(1) 420 else: 421 repository = None 422 423 # Put these into a dictionary, otherwise we cannot assign them 424 # from nested function definitions. 425 d = { 'files': [], 'comments': "" } 426 gobbler = None 427 rev = None 428 who = None 429 when = util.now() 430 def gobble_comment(s): 431 d['comments'] += s + "\n"
432 def gobble_removed(s): 433 d['files'].append('%s REMOVED' % s)
434 def gobble_added(s): 435 d['files'].append('%s ADDED' % s) 436 def gobble_modified(s): 437 d['files'].append('%s MODIFIED' % s) 438 def gobble_renamed(s): 439 match = re.search(r"^(.+) => (.+)$", s) 440 if match: 441 d['files'].append('%s RENAMED %s' % (match.group(1), match.group(2))) 442 else: 443 d['files'].append('%s RENAMED' % s) 444 445 lines = list(body_line_iterator(m, True)) 446 rev = None 447 while lines: 448 line = lines.pop(0) 449 450 # revno: 101 451 match = re.search(r"^revno: ([0-9.]+)", line) 452 if match: 453 rev = match.group(1) 454 455 # committer: Joe <joe@acme.com> 456 match = re.search(r"^committer: (.*)$", line) 457 if match: 458 who = match.group(1) 459 460 # timestamp: Fri 2009-05-15 10:35:43 +0200 461 # datetime.strptime() is supposed to support %z for time zone, but 462 # it does not seem to work. So handle the time zone manually. 463 match = re.search(r"^timestamp: [a-zA-Z]{3} (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) ([-+])(\d{2})(\d{2})$", line) 464 if match: 465 datestr = match.group(1) 466 tz_sign = match.group(2) 467 tz_hours = match.group(3) 468 tz_minutes = match.group(4) 469 when = parseLaunchpadDate(datestr, tz_sign, tz_hours, tz_minutes) 470 471 if re.search(r"^message:\s*$", line): 472 gobbler = gobble_comment 473 elif re.search(r"^removed:\s*$", line): 474 gobbler = gobble_removed 475 elif re.search(r"^added:\s*$", line): 476 gobbler = gobble_added 477 elif re.search(r"^renamed:\s*$", line): 478 gobbler = gobble_renamed 479 elif re.search(r"^modified:\s*$", line): 480 gobbler = gobble_modified 481 elif re.search(r"^ ", line) and gobbler: 482 gobbler(line[2:-1]) # Use :-1 to gobble trailing newline 483 484 # Determine the name of the branch. 485 branch = None 486 if self.branchMap and repository: 487 if self.branchMap.has_key(repository): 488 branch = self.branchMap[repository] 489 elif self.branchMap.has_key('lp:' + repository): 490 branch = self.branchMap['lp:' + repository] 491 if not branch: 492 if self.defaultBranch: 493 branch = self.defaultBranch 494 else: 495 if repository: 496 branch = 'lp:' + repository 497 else: 498 branch = None 499 500 #log.msg("parse(): rev=%s who=%s files=%s comments='%s' when=%s branch=%s" % (rev, who, d['files'], d['comments'], time.asctime(time.localtime(when)), branch)) 501 if rev and who: 502 return changes.Change(who, d['files'], d['comments'], 503 when=when, revision=rev, branch=branch, 504 repository=repository or '') 505 else: 506 return None 507
508 -def parseLaunchpadDate(datestr, tz_sign, tz_hours, tz_minutes):
509 time_no_tz = calendar.timegm(time.strptime(datestr, "%Y-%m-%d %H:%M:%S")) 510 tz_delta = 60 * 60 * int(tz_sign + tz_hours) + 60 * int(tz_minutes) 511 return time_no_tz - tz_delta
512