buildbot.changes.mail

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 """ 17 Parse various kinds of 'CVS notify' email. 18 """ 19 import re 20 import time, calendar 21 import datetime 22 from email import message_from_file 23 from email.Utils import parseaddr, parsedate_tz, mktime_tz 24 from email.Iterators import body_line_iterator 25 26 from zope.interface import implements 27 from twisted.python import log 28 from twisted.internet import defer 29 from buildbot import util 30 from buildbot.interfaces import IChangeSource 31 from buildbot.util.maildir import MaildirService 32

33 -class MaildirSource(MaildirService, util.ComparableMixin):

34 """Generic base class for Maildir-based change sources""" 35 implements(IChangeSource) 36 37 compare_attrs = ["basedir", "pollinterval", "prefix"] 38

39 - def __init__(self, maildir, prefix=None, category='', repository=''):

40 MaildirService.__init__(self, maildir) 41 self.prefix = prefix 42 self.category = category 43 self.repository = repository 44 if prefix and not prefix.endswith("/"): 45 log.msg("%s: you probably want your prefix=('%s') to end with " 46 "a slash")

47

48 - def describe(self):

49 return "%s watching maildir '%s'" % (self.__class__.__name__, self.basedir)

50

51 - def messageReceived(self, filename):

52 d = defer.succeed(None) 53 def parse_file(_): 54 f = self.moveToCurDir(filename) 55 return self.parse_file(f, self.prefix)

56 d.addCallback(parse_file) 57 58 def add_change(chtuple): 59 src, chdict = None, None 60 if chtuple: 61 src, chdict = chtuple 62 if chdict: 63 return self.master.addChange(src=src, **chdict) 64 else: 65 log.msg("no change found in maildir file '%s'" % filename)

66 d.addCallback(add_change) 67 68 return d 69

70 - def parse_file(self, fd, prefix=None):

71 m = message_from_file(fd) 72 return self.parse(m, prefix)

73

74 -class CVSMaildirSource(MaildirSource):

75 name = "CVSMaildirSource" 76

77 - def __init__(self, maildir, prefix=None, category='', 78 repository='', properties={}):

79 MaildirSource.__init__(self, maildir, prefix, category, repository) 80 self.properties = properties

81

82 - def parse(self, m, prefix=None):

83 """Parse messages sent by the 'buildbot-cvs-mail' program. 84 """ 85 # The mail is sent from the person doing the checkin. Assume that the 86 # local username is enough to identify them (this assumes a one-server 87 # cvs-over-rsh environment rather than the server-dirs-shared-over-NFS 88 # model) 89 name, addr = parseaddr(m["from"]) 90 if not addr: 91 return None # no From means this message isn't from buildbot-cvs-mail 92 at = addr.find("@") 93 if at == -1: 94 author = addr # might still be useful 95 else: 96 author = addr[:at] 97 98 # CVS accecpts RFC822 dates. buildbot-cvs-mail adds the date as 99 # part of the mail header, so use that. 100 # This assumes cvs is being access via ssh or pserver, so the time 101 # will be the CVS server's time. 102 103 # calculate a "revision" based on that timestamp, or the current time 104 # if we're unable to parse the date. 105 log.msg('Processing CVS mail') 106 dateTuple = parsedate_tz(m["date"]) 107 if dateTuple == None: 108 when = util.now() 109 else: 110 when = mktime_tz(dateTuple) 111 112 theTime = datetime.datetime.utcfromtimestamp(float(when)) 113 rev = theTime.strftime('%Y-%m-%d %H:%M:%S') 114 115 catRE = re.compile( '^Category:\s*(\S.*)') 116 cvsRE = re.compile( '^CVSROOT:\s*(\S.*)') 117 cvsmodeRE = re.compile( '^Cvsmode:\s*(\S.*)') 118 filesRE = re.compile( '^Files:\s*(\S.*)') 119 modRE = re.compile( '^Module:\s*(\S.*)') 120 pathRE = re.compile( '^Path:\s*(\S.*)') 121 projRE = re.compile( '^Project:\s*(\S.*)') 122 singleFileRE = re.compile( '(.*) (NONE|\d(\.|\d)+) (NONE|\d(\.|\d)+)') 123 tagRE = re.compile( '^\s+Tag:\s*(\S.*)') 124 updateRE = re.compile( '^Update of:\s*(\S.*)') 125 comments = "" 126 branch = None 127 cvsroot = None 128 fileList = None 129 files = [] 130 isdir = 0 131 path = None 132 project = None 133 134 lines = list(body_line_iterator(m)) 135 while lines: 136 line = lines.pop(0) 137 m = catRE.match(line) 138 if m: 139 category = m.group(1) 140 continue 141 m = cvsRE.match(line) 142 if m: 143 cvsroot = m.group(1) 144 continue 145 m = cvsmodeRE.match(line) 146 if m: 147 cvsmode = m.group(1) 148 continue 149 m = filesRE.match(line) 150 if m: 151 fileList = m.group(1) 152 continue 153 m = modRE.match(line) 154 if m: 155 # We don't actually use this 156 #module = m.group(1) 157 continue 158 m = pathRE.match(line) 159 if m: 160 path = m.group(1) 161 continue 162 m = projRE.match(line) 163 if m: 164 project = m.group(1) 165 continue 166 m = tagRE.match(line) 167 if m: 168 branch = m.group(1) 169 continue 170 m = updateRE.match(line) 171 if m: 172 # We don't actually use this 173 #updateof = m.group(1) 174 continue 175 if line == "Log Message:\n": 176 break 177 178 # CVS 1.11 lists files as: 179 # repo/path file,old-version,new-version file2,old-version,new-version 180 # Version 1.12 lists files as: 181 # file1 old-version new-version file2 old-version new-version 182 # 183 # files consists of tuples of 'file-name old-version new-version' 184 # The versions are either dotted-decimal version numbers, ie 1.1 185 # or NONE. New files are of the form 'NONE NUMBER', while removed 186 # files are 'NUMBER NONE'. 'NONE' is a literal string 187 # Parsing this instead of files list in 'Added File:' etc 188 # makes it possible to handle files with embedded spaces, though 189 # it could fail if the filename was 'bad 1.1 1.2' 190 # For cvs version 1.11, we expect 191 # my_module new_file.c,NONE,1.1 192 # my_module removed.txt,1.2,NONE 193 # my_module modified_file.c,1.1,1.2 194 # While cvs version 1.12 gives us 195 # new_file.c NONE 1.1 196 # removed.txt 1.2 NONE 197 # modified_file.c 1.1,1.2 198 199 if fileList is None: 200 log.msg('CVSMaildirSource Mail with no files. Ignoring') 201 return None # We don't have any files. Email not from CVS 202 203 if cvsmode == '1.11': 204 # Please, no repo paths with spaces! 205 m = re.search('([^ ]*) ', fileList) 206 if m: 207 path = m.group(1) 208 else: 209 log.msg('CVSMaildirSource can\'t get path from file list. Ignoring mail') 210 return 211 fileList = fileList[len(path):].strip() 212 singleFileRE = re.compile( '(.+?),(NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+)),(NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+))(?: |$)') 213 elif cvsmode == '1.12': 214 singleFileRE = re.compile( '(.+?) (NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+)) (NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+))(?: |$)') 215 if path is None: 216 raise ValueError('CVSMaildirSource cvs 1.12 require path. Check cvs loginfo config') 217 else: 218 raise ValueError('Expected cvsmode 1.11 or 1.12. got: %s' % cvsmode) 219 220 log.msg("CVSMaildirSource processing filelist: %s" % fileList) 221 links = [] 222 while(fileList): 223 m = singleFileRE.match(fileList) 224 if m: 225 curFile = path + '/' + m.group(1) 226 files.append( curFile ) 227 fileList = fileList[m.end():] 228 else: 229 log.msg('CVSMaildirSource no files matched regex. Ignoring') 230 return None # bail - we couldn't parse the files that changed 231 # Now get comments 232 while lines: 233 line = lines.pop(0) 234 comments += line 235 236 comments = comments.rstrip() + "\n" 237 if comments == '\n': 238 comments = None 239 return ('cvs', dict(author=author, files=files, comments=comments, 240 isdir=isdir, when=when, branch=branch, 241 revision=rev, category=category, 242 repository=cvsroot, project=project, 243 links=links, properties=self.properties))

244 245 # svn "commit-email.pl" handler. The format is very similar to freshcvs mail; 246 # here's a sample: 247 248 # From: username [at] apache.org [slightly obfuscated to avoid spam here] 249 # To: commits [at] spamassassin.apache.org 250 # Subject: svn commit: r105955 - in spamassassin/trunk: . lib/Mail 251 # ... 252 # 253 # Author: username 254 # Date: Sat Nov 20 00:17:49 2004 [note: TZ = local tz on server!] 255 # New Revision: 105955 256 # 257 # Modified: [also Removed: and Added:] 258 # [filename] 259 # ... 260 # Log: 261 # [log message] 262 # ... 263 # 264 # 265 # Modified: spamassassin/trunk/lib/Mail/SpamAssassin.pm 266 # [unified diff] 267 # 268 # [end of mail] 269

270 -class SVNCommitEmailMaildirSource(MaildirSource):

271 name = "SVN commit-email.pl" 272

273 - def parse(self, m, prefix=None):

274 """Parse messages sent by the svn 'commit-email.pl' trigger. 275 """ 276 277 # The mail is sent from the person doing the checkin. Assume that the 278 # local username is enough to identify them (this assumes a one-server 279 # cvs-over-rsh environment rather than the server-dirs-shared-over-NFS 280 # model) 281 name, addr = parseaddr(m["from"]) 282 if not addr: 283 return None # no From means this message isn't from svn 284 at = addr.find("@") 285 if at == -1: 286 author = addr # might still be useful 287 else: 288 author = addr[:at] 289 290 # we take the time of receipt as the time of checkin. Not correct (it 291 # depends upon the email latency), but it avoids the 292 # out-of-order-changes issue. Also syncmail doesn't give us anything 293 # better to work with, unless you count pulling the v1-vs-v2 294 # timestamp out of the diffs, which would be ugly. TODO: Pulling the 295 # 'Date:' header from the mail is a possibility, and 296 # email.Utils.parsedate_tz may be useful. It should be configurable, 297 # however, because there are a lot of broken clocks out there. 298 when = util.now() 299 300 files = [] 301 comments = "" 302 lines = list(body_line_iterator(m)) 303 rev = None 304 while lines: 305 line = lines.pop(0) 306 307 # "Author: jmason" 308 match = re.search(r"^Author: (\S+)", line) 309 if match: 310 author = match.group(1) 311 312 # "New Revision: 105955" 313 match = re.search(r"^New Revision: (\d+)", line) 314 if match: 315 rev = match.group(1) 316 317 # possible TODO: use "Date: ..." data here instead of time of 318 # commit message receipt, above. however, this timestamp is 319 # specified *without* a timezone, in the server's local TZ, so to 320 # be accurate buildbot would need a config setting to specify the 321 # source server's expected TZ setting! messy. 322 323 # this stanza ends with the "Log:" 324 if (line == "Log:\n"): 325 break 326 327 # commit message is terminated by the file-listing section 328 while lines: 329 line = lines.pop(0) 330 if (line == "Modified:\n" or 331 line == "Added:\n" or 332 line == "Removed:\n"): 333 break 334 comments += line 335 comments = comments.rstrip() + "\n" 336 337 while lines: 338 line = lines.pop(0) 339 if line == "\n": 340 break 341 if line.find("Modified:\n") == 0: 342 continue # ignore this line 343 if line.find("Added:\n") == 0: 344 continue # ignore this line 345 if line.find("Removed:\n") == 0: 346 continue # ignore this line 347 line = line.strip() 348 349 thesefiles = line.split(" ") 350 for f in thesefiles: 351 if prefix: 352 # insist that the file start with the prefix: we may get 353 # changes we don't care about too 354 if f.startswith(prefix): 355 f = f[len(prefix):] 356 else: 357 log.msg("ignored file from svn commit: prefix '%s' " 358 "does not match filename '%s'" % (prefix, f)) 359 continue 360 361 # TODO: figure out how new directories are described, set 362 # .isdir 363 files.append(f) 364 365 if not files: 366 log.msg("no matching files found, ignoring commit") 367 return None 368 369 return ('svn', dict(author=author, files=files, comments=comments, 370 when=when, revision=rev))

371 372 # bzr Launchpad branch subscription mails. Sample mail: 373 # 374 # From: noreply@launchpad.net 375 # Subject: [Branch ~knielsen/maria/tmp-buildbot-test] Rev 2701: test add file 376 # To: Joe <joe@acme.com> 377 # ... 378 # 379 # ------------------------------------------------------------ 380 # revno: 2701 381 # committer: Joe <joe@acme.com> 382 # branch nick: tmpbb 383 # timestamp: Fri 2009-05-15 10:35:43 +0200 384 # message: 385 # test add file 386 # added: 387 # test-add-file 388 # 389 # 390 # -- 391 # 392 # https://code.launchpad.net/~knielsen/maria/tmp-buildbot-test 393 # 394 # You are subscribed to branch lp:~knielsen/maria/tmp-buildbot-test. 395 # To unsubscribe from this branch go to https://code.launchpad.net/~knielsen/maria/tmp-buildbot-test/+edit-subscription. 396 # 397 # [end of mail] 398

399 -class BzrLaunchpadEmailMaildirSource(MaildirSource):

400 name = "Launchpad" 401 402 compare_attrs = MaildirSource.compare_attrs + ["branchMap", "defaultBranch"] 403

404 - def __init__(self, maildir, prefix=None, branchMap=None, defaultBranch=None, **kwargs):

405 self.branchMap = branchMap 406 self.defaultBranch = defaultBranch 407 MaildirSource.__init__(self, maildir, prefix, **kwargs)

408

409 - def parse(self, m, prefix=None):

410 """Parse branch notification messages sent by Launchpad. 411 """ 412 413 subject = m["subject"] 414 match = re.search(r"^\s*\[Branch\s+([^]]+)\]", subject) 415 if match: 416 repository = match.group(1) 417 else: 418 repository = None 419 420 # Put these into a dictionary, otherwise we cannot assign them 421 # from nested function definitions. 422 d = { 'files': [], 'comments': u"" } 423 gobbler = None 424 rev = None 425 author = None 426 when = util.now() 427 def gobble_comment(s): 428 d['comments'] += s + "\n"

429 def gobble_removed(s): 430 d['files'].append('%s REMOVED' % s)

431 def gobble_added(s): 432 d['files'].append('%s ADDED' % s) 433 def gobble_modified(s): 434 d['files'].append('%s MODIFIED' % s) 435 def gobble_renamed(s): 436 match = re.search(r"^(.+) => (.+)$", s) 437 if match: 438 d['files'].append('%s RENAMED %s' % (match.group(1), match.group(2))) 439 else: 440 d['files'].append('%s RENAMED' % s) 441 442 lines = list(body_line_iterator(m, True)) 443 rev = None 444 while lines: 445 line = unicode(lines.pop(0), "utf-8", errors="ignore") 446 447 # revno: 101 448 match = re.search(r"^revno: ([0-9.]+)", line) 449 if match: 450 rev = match.group(1) 451 452 # committer: Joe <joe@acme.com> 453 match = re.search(r"^committer: (.*)$", line) 454 if match: 455 author = match.group(1) 456 457 # timestamp: Fri 2009-05-15 10:35:43 +0200 458 # datetime.strptime() is supposed to support %z for time zone, but 459 # it does not seem to work. So handle the time zone manually. 460 match = re.search(r"^timestamp: [a-zA-Z]{3} (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) ([-+])(\d{2})(\d{2})$", line) 461 if match: 462 datestr = match.group(1) 463 tz_sign = match.group(2) 464 tz_hours = match.group(3) 465 tz_minutes = match.group(4) 466 when = parseLaunchpadDate(datestr, tz_sign, tz_hours, tz_minutes) 467 468 if re.search(r"^message:\s*$", line): 469 gobbler = gobble_comment 470 elif re.search(r"^removed:\s*$", line): 471 gobbler = gobble_removed 472 elif re.search(r"^added:\s*$", line): 473 gobbler = gobble_added 474 elif re.search(r"^renamed:\s*$", line): 475 gobbler = gobble_renamed 476 elif re.search(r"^modified:\s*$", line): 477 gobbler = gobble_modified 478 elif re.search(r"^ ", line) and gobbler: 479 gobbler(line[2:-1]) # Use :-1 to gobble trailing newline 480 481 # Determine the name of the branch. 482 branch = None 483 if self.branchMap and repository: 484 if self.branchMap.has_key(repository): 485 branch = self.branchMap[repository] 486 elif self.branchMap.has_key('lp:' + repository): 487 branch = self.branchMap['lp:' + repository] 488 if not branch: 489 if self.defaultBranch: 490 branch = self.defaultBranch 491 else: 492 if repository: 493 branch = 'lp:' + repository 494 else: 495 branch = None 496 497 if rev and author: 498 return ('bzr', dict(author=author, files=d['files'], 499 comments=d['comments'], 500 when=when, revision=rev, 501 branch=branch, repository=repository or '')) 502 else: 503 return None 504

505 -def parseLaunchpadDate(datestr, tz_sign, tz_hours, tz_minutes):

506 time_no_tz = calendar.timegm(time.strptime(datestr, "%Y-%m-%d %H:%M:%S")) 507 tz_delta = 60 * 60 * int(tz_sign + tz_hours) + 60 * int(tz_minutes) 508 return time_no_tz - tz_delta

509

Source Code for Module buildbot.changes.mail