buildbot.changes.mail

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 """ 17 Parse various kinds of 'CVS notify' email. 18 """ 19 import re 20 import time, calendar 21 import datetime 22 from email import message_from_file 23 from email.Utils import parseaddr, parsedate_tz, mktime_tz 24 from email.Iterators import body_line_iterator 25 26 from zope.interface import implements 27 from twisted.python import log 28 from twisted.internet import defer 29 from buildbot import util 30 from buildbot.interfaces import IChangeSource 31 from buildbot.util.maildir import MaildirService 32

33 -class MaildirSource(MaildirService, util.ComparableMixin):

34 """Generic base class for Maildir-based change sources""" 35 implements(IChangeSource) 36 37 compare_attrs = ["basedir", "pollinterval", "prefix"] 38

39 - def __init__(self, maildir, prefix=None, category='', repository=''):

40 MaildirService.__init__(self, maildir) 41 self.prefix = prefix 42 self.category = category 43 self.repository = repository 44 if prefix and not prefix.endswith("/"): 45 log.msg("%s: you probably want your prefix=('%s') to end with " 46 "a slash")

47

48 - def describe(self):

49 return "%s watching maildir '%s'" % (self.__class__.__name__, self.basedir)

50

51 - def messageReceived(self, filename):

52 d = defer.succeed(None) 53 def parse_file(_): 54 f = self.moveToCurDir(filename) 55 return self.parse_file(f, self.prefix)

56 d.addCallback(parse_file) 57 58 def add_change(chtuple): 59 src, chdict = None, None 60 if chtuple: 61 src, chdict = chtuple 62 if chdict: 63 return self.master.addChange(src=src, **chdict) 64 else: 65 log.msg("no change found in maildir file '%s'" % filename)

66 d.addCallback(add_change) 67 68 return d 69

70 - def parse_file(self, fd, prefix=None):

71 m = message_from_file(fd) 72 return self.parse(m, prefix)

73

74 -class CVSMaildirSource(MaildirSource):

75 name = "CVSMaildirSource" 76

77 - def __init__(self, maildir, prefix=None, category='', 78 repository='', properties={}):

79 MaildirSource.__init__(self, maildir, prefix, category, repository) 80 self.properties = properties

81

82 - def parse(self, m, prefix=None):

83 """Parse messages sent by the 'buildbot-cvs-mail' program. 84 """ 85 # The mail is sent from the person doing the checkin. Assume that the 86 # local username is enough to identify them (this assumes a one-server 87 # cvs-over-rsh environment rather than the server-dirs-shared-over-NFS 88 # model) 89 name, addr = parseaddr(m["from"]) 90 if not addr: 91 return None # no From means this message isn't from buildbot-cvs-mail 92 at = addr.find("@") 93 if at == -1: 94 author = addr # might still be useful 95 else: 96 author = addr[:at] 97 98 # CVS accecpts RFC822 dates. buildbot-cvs-mail adds the date as 99 # part of the mail header, so use that. 100 # This assumes cvs is being access via ssh or pserver, so the time 101 # will be the CVS server's time. 102 103 # calculate a "revision" based on that timestamp, or the current time 104 # if we're unable to parse the date. 105 log.msg('Processing CVS mail') 106 dateTuple = parsedate_tz(m["date"]) 107 if dateTuple == None: 108 when = util.now() 109 else: 110 when = mktime_tz(dateTuple) 111 112 theTime = datetime.datetime.utcfromtimestamp(float(when)) 113 rev = theTime.strftime('%Y-%m-%d %H:%M:%S') 114 115 catRE = re.compile( '^Category:\s*(\S.*)') 116 cvsRE = re.compile( '^CVSROOT:\s*(\S.*)') 117 cvsmodeRE = re.compile( '^Cvsmode:\s*(\S.*)') 118 filesRE = re.compile( '^Files:\s*(\S.*)') 119 modRE = re.compile( '^Module:\s*(\S.*)') 120 pathRE = re.compile( '^Path:\s*(\S.*)') 121 projRE = re.compile( '^Project:\s*(\S.*)') 122 singleFileRE = re.compile( '(.*) (NONE|\d(\.|\d)+) (NONE|\d(\.|\d)+)') 123 tagRE = re.compile( '^\s+Tag:\s*(\S.*)') 124 updateRE = re.compile( '^Update of:\s*(\S.*)') 125 comments = "" 126 branch = None 127 cvsroot = None 128 fileList = None 129 files = [] 130 isdir = 0 131 path = None 132 project = None 133 134 lines = list(body_line_iterator(m)) 135 while lines: 136 line = lines.pop(0) 137 m = catRE.match(line) 138 if m: 139 category = m.group(1) 140 continue 141 m = cvsRE.match(line) 142 if m: 143 cvsroot = m.group(1) 144 continue 145 m = cvsmodeRE.match(line) 146 if m: 147 cvsmode = m.group(1) 148 continue 149 m = filesRE.match(line) 150 if m: 151 fileList = m.group(1) 152 continue 153 m = modRE.match(line) 154 if m: 155 # We don't actually use this 156 #module = m.group(1) 157 continue 158 m = pathRE.match(line) 159 if m: 160 path = m.group(1) 161 continue 162 m = projRE.match(line) 163 if m: 164 project = m.group(1) 165 continue 166 m = tagRE.match(line) 167 if m: 168 branch = m.group(1) 169 continue 170 m = updateRE.match(line) 171 if m: 172 # We don't actually use this 173 #updateof = m.group(1) 174 continue 175 if line == "Log Message:\n": 176 break 177 178 # CVS 1.11 lists files as: 179 # repo/path file,old-version,new-version file2,old-version,new-version 180 # Version 1.12 lists files as: 181 # file1 old-version new-version file2 old-version new-version 182 # 183 # files consists of tuples of 'file-name old-version new-version' 184 # The versions are either dotted-decimal version numbers, ie 1.1 185 # or NONE. New files are of the form 'NONE NUMBER', while removed 186 # files are 'NUMBER NONE'. 'NONE' is a literal string 187 # Parsing this instead of files list in 'Added File:' etc 188 # makes it possible to handle files with embedded spaces, though 189 # it could fail if the filename was 'bad 1.1 1.2' 190 # For cvs version 1.11, we expect 191 # my_module new_file.c,NONE,1.1 192 # my_module removed.txt,1.2,NONE 193 # my_module modified_file.c,1.1,1.2 194 # While cvs version 1.12 gives us 195 # new_file.c NONE 1.1 196 # removed.txt 1.2 NONE 197 # modified_file.c 1.1,1.2 198 199 if fileList is None: 200 log.msg('CVSMaildirSource Mail with no files. Ignoring') 201 return None # We don't have any files. Email not from CVS 202 203 if cvsmode == '1.11': 204 # Please, no repo paths with spaces! 205 m = re.search('([^ ]*) ', fileList) 206 if m: 207 path = m.group(1) 208 else: 209 log.msg('CVSMaildirSource can\'t get path from file list. Ignoring mail') 210 return 211 fileList = fileList[len(path):].strip() 212 singleFileRE = re.compile( '(.+?),(NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+)),(NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+))(?: |$)') 213 elif cvsmode == '1.12': 214 singleFileRE = re.compile( '(.+?) (NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+)) (NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+))(?: |$)') 215 if path is None: 216 raise ValueError('CVSMaildirSource cvs 1.12 require path. Check cvs loginfo config') 217 else: 218 raise ValueError('Expected cvsmode 1.11 or 1.12. got: %s' % cvsmode) 219 220 log.msg("CVSMaildirSource processing filelist: %s" % fileList) 221 while(fileList): 222 m = singleFileRE.match(fileList) 223 if m: 224 curFile = path + '/' + m.group(1) 225 files.append( curFile ) 226 fileList = fileList[m.end():] 227 else: 228 log.msg('CVSMaildirSource no files matched regex. Ignoring') 229 return None # bail - we couldn't parse the files that changed 230 # Now get comments 231 while lines: 232 line = lines.pop(0) 233 comments += line 234 235 comments = comments.rstrip() + "\n" 236 if comments == '\n': 237 comments = None 238 return ('cvs', dict(author=author, files=files, comments=comments, 239 isdir=isdir, when=when, branch=branch, 240 revision=rev, category=category, 241 repository=cvsroot, project=project, 242 properties=self.properties))

243 244 # svn "commit-email.pl" handler. The format is very similar to freshcvs mail; 245 # here's a sample: 246 247 # From: username [at] apache.org [slightly obfuscated to avoid spam here] 248 # To: commits [at] spamassassin.apache.org 249 # Subject: svn commit: r105955 - in spamassassin/trunk: . lib/Mail 250 # ... 251 # 252 # Author: username 253 # Date: Sat Nov 20 00:17:49 2004 [note: TZ = local tz on server!] 254 # New Revision: 105955 255 # 256 # Modified: [also Removed: and Added:] 257 # [filename] 258 # ... 259 # Log: 260 # [log message] 261 # ... 262 # 263 # 264 # Modified: spamassassin/trunk/lib/Mail/SpamAssassin.pm 265 # [unified diff] 266 # 267 # [end of mail] 268

269 -class SVNCommitEmailMaildirSource(MaildirSource):

270 name = "SVN commit-email.pl" 271

272 - def parse(self, m, prefix=None):

273 """Parse messages sent by the svn 'commit-email.pl' trigger. 274 """ 275 276 # The mail is sent from the person doing the checkin. Assume that the 277 # local username is enough to identify them (this assumes a one-server 278 # cvs-over-rsh environment rather than the server-dirs-shared-over-NFS 279 # model) 280 name, addr = parseaddr(m["from"]) 281 if not addr: 282 return None # no From means this message isn't from svn 283 at = addr.find("@") 284 if at == -1: 285 author = addr # might still be useful 286 else: 287 author = addr[:at] 288 289 # we take the time of receipt as the time of checkin. Not correct (it 290 # depends upon the email latency), but it avoids the 291 # out-of-order-changes issue. Also syncmail doesn't give us anything 292 # better to work with, unless you count pulling the v1-vs-v2 293 # timestamp out of the diffs, which would be ugly. TODO: Pulling the 294 # 'Date:' header from the mail is a possibility, and 295 # email.Utils.parsedate_tz may be useful. It should be configurable, 296 # however, because there are a lot of broken clocks out there. 297 when = util.now() 298 299 files = [] 300 comments = "" 301 lines = list(body_line_iterator(m)) 302 rev = None 303 while lines: 304 line = lines.pop(0) 305 306 # "Author: jmason" 307 match = re.search(r"^Author: (\S+)", line) 308 if match: 309 author = match.group(1) 310 311 # "New Revision: 105955" 312 match = re.search(r"^New Revision: (\d+)", line) 313 if match: 314 rev = match.group(1) 315 316 # possible TODO: use "Date: ..." data here instead of time of 317 # commit message receipt, above. however, this timestamp is 318 # specified *without* a timezone, in the server's local TZ, so to 319 # be accurate buildbot would need a config setting to specify the 320 # source server's expected TZ setting! messy. 321 322 # this stanza ends with the "Log:" 323 if (line == "Log:\n"): 324 break 325 326 # commit message is terminated by the file-listing section 327 while lines: 328 line = lines.pop(0) 329 if (line == "Modified:\n" or 330 line == "Added:\n" or 331 line == "Removed:\n"): 332 break 333 comments += line 334 comments = comments.rstrip() + "\n" 335 336 while lines: 337 line = lines.pop(0) 338 if line == "\n": 339 break 340 if line.find("Modified:\n") == 0: 341 continue # ignore this line 342 if line.find("Added:\n") == 0: 343 continue # ignore this line 344 if line.find("Removed:\n") == 0: 345 continue # ignore this line 346 line = line.strip() 347 348 thesefiles = line.split(" ") 349 for f in thesefiles: 350 if prefix: 351 # insist that the file start with the prefix: we may get 352 # changes we don't care about too 353 if f.startswith(prefix): 354 f = f[len(prefix):] 355 else: 356 log.msg("ignored file from svn commit: prefix '%s' " 357 "does not match filename '%s'" % (prefix, f)) 358 continue 359 360 # TODO: figure out how new directories are described, set 361 # .isdir 362 files.append(f) 363 364 if not files: 365 log.msg("no matching files found, ignoring commit") 366 return None 367 368 return ('svn', dict(author=author, files=files, comments=comments, 369 when=when, revision=rev))

370 371 # bzr Launchpad branch subscription mails. Sample mail: 372 # 373 # From: noreply@launchpad.net 374 # Subject: [Branch ~knielsen/maria/tmp-buildbot-test] Rev 2701: test add file 375 # To: Joe <joe@acme.com> 376 # ... 377 # 378 # ------------------------------------------------------------ 379 # revno: 2701 380 # committer: Joe <joe@acme.com> 381 # branch nick: tmpbb 382 # timestamp: Fri 2009-05-15 10:35:43 +0200 383 # message: 384 # test add file 385 # added: 386 # test-add-file 387 # 388 # 389 # -- 390 # 391 # https://code.launchpad.net/~knielsen/maria/tmp-buildbot-test 392 # 393 # You are subscribed to branch lp:~knielsen/maria/tmp-buildbot-test. 394 # To unsubscribe from this branch go to https://code.launchpad.net/~knielsen/maria/tmp-buildbot-test/+edit-subscription. 395 # 396 # [end of mail] 397

398 -class BzrLaunchpadEmailMaildirSource(MaildirSource):

399 name = "Launchpad" 400 401 compare_attrs = MaildirSource.compare_attrs + ["branchMap", "defaultBranch"] 402

403 - def __init__(self, maildir, prefix=None, branchMap=None, defaultBranch=None, **kwargs):

404 self.branchMap = branchMap 405 self.defaultBranch = defaultBranch 406 MaildirSource.__init__(self, maildir, prefix, **kwargs)

407

408 - def parse(self, m, prefix=None):

409 """Parse branch notification messages sent by Launchpad. 410 """ 411 412 subject = m["subject"] 413 match = re.search(r"^\s*\[Branch\s+([^]]+)\]", subject) 414 if match: 415 repository = match.group(1) 416 else: 417 repository = None 418 419 # Put these into a dictionary, otherwise we cannot assign them 420 # from nested function definitions. 421 d = { 'files': [], 'comments': u"" } 422 gobbler = None 423 rev = None 424 author = None 425 when = util.now() 426 def gobble_comment(s): 427 d['comments'] += s + "\n"

428 def gobble_removed(s): 429 d['files'].append('%s REMOVED' % s)

430 def gobble_added(s): 431 d['files'].append('%s ADDED' % s) 432 def gobble_modified(s): 433 d['files'].append('%s MODIFIED' % s) 434 def gobble_renamed(s): 435 match = re.search(r"^(.+) => (.+)$", s) 436 if match: 437 d['files'].append('%s RENAMED %s' % (match.group(1), match.group(2))) 438 else: 439 d['files'].append('%s RENAMED' % s) 440 441 lines = list(body_line_iterator(m, True)) 442 rev = None 443 while lines: 444 line = unicode(lines.pop(0), "utf-8", errors="ignore") 445 446 # revno: 101 447 match = re.search(r"^revno: ([0-9.]+)", line) 448 if match: 449 rev = match.group(1) 450 451 # committer: Joe <joe@acme.com> 452 match = re.search(r"^committer: (.*)$", line) 453 if match: 454 author = match.group(1) 455 456 # timestamp: Fri 2009-05-15 10:35:43 +0200 457 # datetime.strptime() is supposed to support %z for time zone, but 458 # it does not seem to work. So handle the time zone manually. 459 match = re.search(r"^timestamp: [a-zA-Z]{3} (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) ([-+])(\d{2})(\d{2})$", line) 460 if match: 461 datestr = match.group(1) 462 tz_sign = match.group(2) 463 tz_hours = match.group(3) 464 tz_minutes = match.group(4) 465 when = parseLaunchpadDate(datestr, tz_sign, tz_hours, tz_minutes) 466 467 if re.search(r"^message:\s*$", line): 468 gobbler = gobble_comment 469 elif re.search(r"^removed:\s*$", line): 470 gobbler = gobble_removed 471 elif re.search(r"^added:\s*$", line): 472 gobbler = gobble_added 473 elif re.search(r"^renamed:\s*$", line): 474 gobbler = gobble_renamed 475 elif re.search(r"^modified:\s*$", line): 476 gobbler = gobble_modified 477 elif re.search(r"^ ", line) and gobbler: 478 gobbler(line[2:-1]) # Use :-1 to gobble trailing newline 479 480 # Determine the name of the branch. 481 branch = None 482 if self.branchMap and repository: 483 if self.branchMap.has_key(repository): 484 branch = self.branchMap[repository] 485 elif self.branchMap.has_key('lp:' + repository): 486 branch = self.branchMap['lp:' + repository] 487 if not branch: 488 if self.defaultBranch: 489 branch = self.defaultBranch 490 else: 491 if repository: 492 branch = 'lp:' + repository 493 else: 494 branch = None 495 496 if rev and author: 497 return ('bzr', dict(author=author, files=d['files'], 498 comments=d['comments'], 499 when=when, revision=rev, 500 branch=branch, repository=repository or '')) 501 else: 502 return None 503

504 -def parseLaunchpadDate(datestr, tz_sign, tz_hours, tz_minutes):

505 time_no_tz = calendar.timegm(time.strptime(datestr, "%Y-%m-%d %H:%M:%S")) 506 tz_delta = 60 * 60 * int(tz_sign + tz_hours) + 60 * int(tz_minutes) 507 return time_no_tz - tz_delta

508

Source Code for Module buildbot.changes.mail