buildbot.changes.mail

1 # -*- test-case-name: buildbot.test.test_mailparse -*- 2 3 """ 4 Parse various kinds of 'CVS notify' email. 5 """ 6 import os, re 7 import time, calendar 8 import datetime 9 from email import message_from_file 10 from email.Utils import parseaddr 11 from email.Iterators import body_line_iterator 12 13 from zope.interface import implements 14 from twisted.python import log 15 from buildbot import util 16 from buildbot.interfaces import IChangeSource 17 from buildbot.changes import changes 18 from buildbot.changes.maildir import MaildirService 19

20 -class MaildirSource(MaildirService, util.ComparableMixin):

21 """This source will watch a maildir that is subscribed to a FreshCVS 22 change-announcement mailing list. 23 """ 24 implements(IChangeSource) 25 26 compare_attrs = ["basedir", "pollinterval", "prefix"] 27 name = None 28

29 - def __init__(self, maildir, prefix=None):

30 MaildirService.__init__(self, maildir) 31 self.prefix = prefix 32 if prefix and not prefix.endswith("/"): 33 log.msg("%s: you probably want your prefix=('%s') to end with " 34 "a slash")

35

36 - def describe(self):

37 return "%s mailing list in maildir %s" % (self.name, self.basedir)

38

39 - def messageReceived(self, filename):

40 path = os.path.join(self.basedir, "new", filename) 41 change = self.parse_file(open(path, "r"), self.prefix) 42 if change: 43 self.parent.addChange(change) 44 os.rename(os.path.join(self.basedir, "new", filename), 45 os.path.join(self.basedir, "cur", filename))

46

47 - def parse_file(self, fd, prefix=None):

48 m = message_from_file(fd) 49 return self.parse(m, prefix)

50

51 -class FCMaildirSource(MaildirSource):

52 name = "FreshCVS" 53

54 - def parse(self, m, prefix=None):

55 """Parse mail sent by FreshCVS""" 56 57 # FreshCVS sets From: to "user CVS <user>", but the <> part may be 58 # modified by the MTA (to include a local domain) 59 name, addr = parseaddr(m["from"]) 60 if not name: 61 return None # no From means this message isn't from FreshCVS 62 cvs = name.find(" CVS") 63 if cvs == -1: 64 return None # this message isn't from FreshCVS 65 who = name[:cvs] 66 67 # we take the time of receipt as the time of checkin. Not correct, 68 # but it avoids the out-of-order-changes issue. See the comment in 69 # parseSyncmail about using the 'Date:' header 70 when = util.now() 71 72 files = [] 73 comments = "" 74 isdir = 0 75 lines = list(body_line_iterator(m)) 76 while lines: 77 line = lines.pop(0) 78 if line == "Modified files:\n": 79 break 80 while lines: 81 line = lines.pop(0) 82 if line == "\n": 83 break 84 line = line.rstrip("\n") 85 linebits = line.split(None, 1) 86 file = linebits[0] 87 if prefix: 88 # insist that the file start with the prefix: FreshCVS sends 89 # changes we don't care about too 90 if file.startswith(prefix): 91 file = file[len(prefix):] 92 else: 93 continue 94 if len(linebits) == 1: 95 isdir = 1 96 elif linebits[1] == "0 0": 97 isdir = 1 98 files.append(file) 99 while lines: 100 line = lines.pop(0) 101 if line == "Log message:\n": 102 break 103 # message is terminated by "ViewCVS links:" or "Index:..." (patch) 104 while lines: 105 line = lines.pop(0) 106 if line == "ViewCVS links:\n": 107 break 108 if line.find("Index: ") == 0: 109 break 110 comments += line 111 comments = comments.rstrip() + "\n" 112 113 if not files: 114 return None 115 116 change = changes.Change(who, files, comments, isdir, when=when) 117 118 return change

119

120 -class SyncmailMaildirSource(MaildirSource):

121 name = "Syncmail" 122

123 - def parse(self, m, prefix=None):

124 """Parse messages sent by the 'syncmail' program, as suggested by the 125 sourceforge.net CVS Admin documentation. Syncmail is maintained at 126 syncmail.sf.net . 127 """ 128 # pretty much the same as freshcvs mail, not surprising since CVS is 129 # the one creating most of the text 130 131 # The mail is sent from the person doing the checkin. Assume that the 132 # local username is enough to identify them (this assumes a one-server 133 # cvs-over-rsh environment rather than the server-dirs-shared-over-NFS 134 # model) 135 name, addr = parseaddr(m["from"]) 136 if not addr: 137 return None # no From means this message isn't from FreshCVS 138 at = addr.find("@") 139 if at == -1: 140 who = addr # might still be useful 141 else: 142 who = addr[:at] 143 144 # we take the time of receipt as the time of checkin. Not correct (it 145 # depends upon the email latency), but it avoids the 146 # out-of-order-changes issue. Also syncmail doesn't give us anything 147 # better to work with, unless you count pulling the v1-vs-v2 148 # timestamp out of the diffs, which would be ugly. TODO: Pulling the 149 # 'Date:' header from the mail is a possibility, and 150 # email.Utils.parsedate_tz may be useful. It should be configurable, 151 # however, because there are a lot of broken clocks out there. 152 when = util.now() 153 154 # calculate a "revision" based on that timestamp 155 theCurrentTime = datetime.datetime.utcfromtimestamp(float(when)) 156 rev = theCurrentTime.isoformat() 157 158 subject = m["subject"] 159 # syncmail puts the repository-relative directory in the subject: 160 # mprefix + "%(dir)s %(file)s,%(oldversion)s,%(newversion)s", where 161 # 'mprefix' is something that could be added by a mailing list 162 # manager. 163 # this is the only reasonable way to determine the directory name 164 space = subject.find(" ") 165 if space != -1: 166 directory = subject[:space] 167 else: 168 directory = subject 169 170 files = [] 171 comments = "" 172 isdir = 0 173 branch = None 174 175 lines = list(body_line_iterator(m)) 176 while lines: 177 line = lines.pop(0) 178 179 if (line == "Modified Files:\n" or 180 line == "Added Files:\n" or 181 line == "Removed Files:\n"): 182 break 183 184 while lines: 185 line = lines.pop(0) 186 if line == "\n": 187 break 188 if line == "Log Message:\n": 189 lines.insert(0, line) 190 break 191 line = line.lstrip() 192 line = line.rstrip() 193 # note: syncmail will send one email per directory involved in a 194 # commit, with multiple files if they were in the same directory. 195 # Unlike freshCVS, it makes no attempt to collect all related 196 # commits into a single message. 197 198 # note: syncmail will report a Tag underneath the ... Files: line 199 # e.g.: Tag: BRANCH-DEVEL 200 201 if line.startswith('Tag:'): 202 branch = line.split(' ')[-1].rstrip() 203 continue 204 205 thesefiles = line.split(" ") 206 for f in thesefiles: 207 f = directory + "/" + f 208 if prefix: 209 # insist that the file start with the prefix: we may get 210 # changes we don't care about too 211 if f.startswith(prefix): 212 f = f[len(prefix):] 213 else: 214 continue 215 break 216 # TODO: figure out how new directories are described, set 217 # .isdir 218 files.append(f) 219 220 if not files: 221 return None 222 223 while lines: 224 line = lines.pop(0) 225 if line == "Log Message:\n": 226 break 227 # message is terminated by "Index:..." (patch) or "--- NEW FILE.." 228 # or "--- filename DELETED ---". Sigh. 229 while lines: 230 line = lines.pop(0) 231 if line.find("Index: ") == 0: 232 break 233 if re.search(r"^--- NEW FILE", line): 234 break 235 if re.search(r" DELETED ---$", line): 236 break 237 comments += line 238 comments = comments.rstrip() + "\n" 239 240 change = changes.Change(who, files, comments, isdir, when=when, 241 branch=branch, revision=rev) 242 243 return change

244 245 # Bonsai mail parser by Stephen Davis. 246 # 247 # This handles changes for CVS repositories that are watched by Bonsai 248 # (http://www.mozilla.org/bonsai.html) 249 250 # A Bonsai-formatted email message looks like: 251 # 252 # C|1071099907|stephend|/cvs|Sources/Scripts/buildbot|bonsai.py|1.2|||18|7 253 # A|1071099907|stephend|/cvs|Sources/Scripts/buildbot|master.cfg|1.1|||18|7 254 # R|1071099907|stephend|/cvs|Sources/Scripts/buildbot|BuildMaster.py||| 255 # LOGCOMMENT 256 # Updated bonsai parser and switched master config to buildbot-0.4.1 style. 257 # 258 # :ENDLOGCOMMENT 259 # 260 # In the first example line, stephend is the user, /cvs the repository, 261 # buildbot the directory, bonsai.py the file, 1.2 the revision, no sticky 262 # and branch, 18 lines added and 7 removed. All of these fields might not be 263 # present (during "removes" for example). 264 # 265 # There may be multiple "control" lines or even none (imports, directory 266 # additions) but there is one email per directory. We only care about actual 267 # changes since it is presumed directory additions don't actually affect the 268 # build. At least one file should need to change (the makefile, say) to 269 # actually make a new directory part of the build process. That's my story 270 # and I'm sticking to it. 271

272 -class BonsaiMaildirSource(MaildirSource):

273 name = "Bonsai" 274

275 - def parse(self, m, prefix=None):

276 """Parse mail sent by the Bonsai cvs loginfo script.""" 277 278 # we don't care who the email came from b/c the cvs user is in the 279 # msg text 280 281 who = "unknown" 282 timestamp = None 283 files = [] 284 lines = list(body_line_iterator(m)) 285 286 # read the control lines (what/who/where/file/etc.) 287 while lines: 288 line = lines.pop(0) 289 if line == "LOGCOMMENT\n": 290 break; 291 line = line.rstrip("\n") 292 293 # we'd like to do the following but it won't work if the number of 294 # items doesn't match so... 295 # what, timestamp, user, repo, module, file = line.split( '|' ) 296 items = line.split('|') 297 if len(items) < 6: 298 # not a valid line, assume this isn't a bonsai message 299 return None 300 301 try: 302 # just grab the bottom-most timestamp, they're probably all the 303 # same. TODO: I'm assuming this is relative to the epoch, but 304 # this needs testing. 305 timestamp = int(items[1]) 306 except ValueError: 307 pass 308 309 user = items[2] 310 if user: 311 who = user 312 313 module = items[4] 314 file = items[5] 315 if module and file: 316 path = "%s/%s" % (module, file) 317 files.append(path) 318 sticky = items[7] 319 branch = items[8] 320 321 # if no files changed, return nothing 322 if not files: 323 return None 324 325 # read the comments 326 comments = "" 327 while lines: 328 line = lines.pop(0) 329 if line == ":ENDLOGCOMMENT\n": 330 break 331 comments += line 332 comments = comments.rstrip() + "\n" 333 334 # return buildbot Change object 335 return changes.Change(who, files, comments, when=timestamp, 336 branch=branch)

337 338 # svn "commit-email.pl" handler. The format is very similar to freshcvs mail; 339 # here's a sample: 340 341 # From: username [at] apache.org [slightly obfuscated to avoid spam here] 342 # To: commits [at] spamassassin.apache.org 343 # Subject: svn commit: r105955 - in spamassassin/trunk: . lib/Mail 344 # ... 345 # 346 # Author: username 347 # Date: Sat Nov 20 00:17:49 2004 [note: TZ = local tz on server!] 348 # New Revision: 105955 349 # 350 # Modified: [also Removed: and Added:] 351 # [filename] 352 # ... 353 # Log: 354 # [log message] 355 # ... 356 # 357 # 358 # Modified: spamassassin/trunk/lib/Mail/SpamAssassin.pm 359 # [unified diff] 360 # 361 # [end of mail] 362

363 -class SVNCommitEmailMaildirSource(MaildirSource):

364 name = "SVN commit-email.pl" 365

366 - def parse(self, m, prefix=None):

367 """Parse messages sent by the svn 'commit-email.pl' trigger. 368 """ 369 370 # The mail is sent from the person doing the checkin. Assume that the 371 # local username is enough to identify them (this assumes a one-server 372 # cvs-over-rsh environment rather than the server-dirs-shared-over-NFS 373 # model) 374 name, addr = parseaddr(m["from"]) 375 if not addr: 376 return None # no From means this message isn't from FreshCVS 377 at = addr.find("@") 378 if at == -1: 379 who = addr # might still be useful 380 else: 381 who = addr[:at] 382 383 # we take the time of receipt as the time of checkin. Not correct (it 384 # depends upon the email latency), but it avoids the 385 # out-of-order-changes issue. Also syncmail doesn't give us anything 386 # better to work with, unless you count pulling the v1-vs-v2 387 # timestamp out of the diffs, which would be ugly. TODO: Pulling the 388 # 'Date:' header from the mail is a possibility, and 389 # email.Utils.parsedate_tz may be useful. It should be configurable, 390 # however, because there are a lot of broken clocks out there. 391 when = util.now() 392 393 files = [] 394 comments = "" 395 isdir = 0 396 lines = list(body_line_iterator(m)) 397 rev = None 398 while lines: 399 line = lines.pop(0) 400 401 # "Author: jmason" 402 match = re.search(r"^Author: (\S+)", line) 403 if match: 404 who = match.group(1) 405 406 # "New Revision: 105955" 407 match = re.search(r"^New Revision: (\d+)", line) 408 if match: 409 rev = match.group(1) 410 411 # possible TODO: use "Date: ..." data here instead of time of 412 # commit message receipt, above. however, this timestamp is 413 # specified *without* a timezone, in the server's local TZ, so to 414 # be accurate buildbot would need a config setting to specify the 415 # source server's expected TZ setting! messy. 416 417 # this stanza ends with the "Log:" 418 if (line == "Log:\n"): 419 break 420 421 # commit message is terminated by the file-listing section 422 while lines: 423 line = lines.pop(0) 424 if (line == "Modified:\n" or 425 line == "Added:\n" or 426 line == "Removed:\n"): 427 break 428 comments += line 429 comments = comments.rstrip() + "\n" 430 431 while lines: 432 line = lines.pop(0) 433 if line == "\n": 434 break 435 if line.find("Modified:\n") == 0: 436 continue # ignore this line 437 if line.find("Added:\n") == 0: 438 continue # ignore this line 439 if line.find("Removed:\n") == 0: 440 continue # ignore this line 441 line = line.strip() 442 443 thesefiles = line.split(" ") 444 for f in thesefiles: 445 if prefix: 446 # insist that the file start with the prefix: we may get 447 # changes we don't care about too 448 if f.startswith(prefix): 449 f = f[len(prefix):] 450 else: 451 log.msg("ignored file from svn commit: prefix '%s' " 452 "does not match filename '%s'" % (prefix, f)) 453 continue 454 455 # TODO: figure out how new directories are described, set 456 # .isdir 457 files.append(f) 458 459 if not files: 460 log.msg("no matching files found, ignoring commit") 461 return None 462 463 return changes.Change(who, files, comments, when=when, revision=rev)

464 465 # bzr Launchpad branch subscription mails. Sample mail: 466 # 467 # From: noreply@launchpad.net 468 # Subject: [Branch ~knielsen/maria/tmp-buildbot-test] Rev 2701: test add file 469 # To: Joe <joe@acme.com> 470 # ... 471 # 472 # ------------------------------------------------------------ 473 # revno: 2701 474 # committer: Joe <joe@acme.com> 475 # branch nick: tmpbb 476 # timestamp: Fri 2009-05-15 10:35:43 +0200 477 # message: 478 # test add file 479 # added: 480 # test-add-file 481 # 482 # 483 # -- 484 # 485 # https://code.launchpad.net/~knielsen/maria/tmp-buildbot-test 486 # 487 # You are subscribed to branch lp:~knielsen/maria/tmp-buildbot-test. 488 # To unsubscribe from this branch go to https://code.launchpad.net/~knielsen/maria/tmp-buildbot-test/+edit-subscription. 489 # 490 # [end of mail] 491

492 -class BzrLaunchpadEmailMaildirSource(MaildirSource):

493 name = "Launchpad" 494 495 compare_attrs = MaildirSource.compare_attrs + ["branchMap", "defaultBranch"] 496

497 - def __init__(self, maildir, prefix=None, branchMap=None, defaultBranch=None, **kwargs):

498 self.branchMap = branchMap 499 self.defaultBranch = defaultBranch 500 MaildirSource.__init__(self, maildir, prefix, **kwargs)

501

502 - def parse(self, m, prefix=None):

503 """Parse branch notification messages sent by Launchpad. 504 """ 505 506 subject = m["subject"] 507 match = re.search(r"^\s*\[Branch\s+([^]]+)\]", subject) 508 if match: 509 repository = match.group(1) 510 else: 511 repository = None 512 513 # Put these into a dictionary, otherwise we cannot assign them 514 # from nested function definitions. 515 d = { 'files': [], 'comments': "" } 516 gobbler = None 517 rev = None 518 who = None 519 when = util.now() 520 def gobble_comment(s): 521 d['comments'] += s + "\n"

522 def gobble_removed(s): 523 d['files'].append('%s REMOVED' % s)

524 def gobble_added(s): 525 d['files'].append('%s ADDED' % s) 526 def gobble_modified(s): 527 d['files'].append('%s MODIFIED' % s) 528 def gobble_renamed(s): 529 match = re.search(r"^(.+) => (.+)$", s) 530 if match: 531 d['files'].append('%s RENAMED %s' % (match.group(1), match.group(2))) 532 else: 533 d['files'].append('%s RENAMED' % s) 534 535 lines = list(body_line_iterator(m, True)) 536 rev = None 537 while lines: 538 line = lines.pop(0) 539 540 # revno: 101 541 match = re.search(r"^revno: ([0-9.]+)", line) 542 if match: 543 rev = match.group(1) 544 545 # committer: Joe <joe@acme.com> 546 match = re.search(r"^committer: (.*)$", line) 547 if match: 548 who = match.group(1) 549 550 # timestamp: Fri 2009-05-15 10:35:43 +0200 551 # datetime.strptime() is supposed to support %z for time zone, but 552 # it does not seem to work. So handle the time zone manually. 553 match = re.search(r"^timestamp: [a-zA-Z]{3} (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) ([-+])(\d{2})(\d{2})$", line) 554 if match: 555 datestr = match.group(1) 556 tz_sign = match.group(2) 557 tz_hours = match.group(3) 558 tz_minutes = match.group(4) 559 when = parseLaunchpadDate(datestr, tz_sign, tz_hours, tz_minutes) 560 561 if re.search(r"^message:\s*$", line): 562 gobbler = gobble_comment 563 elif re.search(r"^removed:\s*$", line): 564 gobbler = gobble_removed 565 elif re.search(r"^added:\s*$", line): 566 gobbler = gobble_added 567 elif re.search(r"^renamed:\s*$", line): 568 gobbler = gobble_renamed 569 elif re.search(r"^modified:\s*$", line): 570 gobbler = gobble_modified 571 elif re.search(r"^ ", line) and gobbler: 572 gobbler(line[2:-1]) # Use :-1 to gobble trailing newline 573 574 # Determine the name of the branch. 575 branch = None 576 if self.branchMap and repository: 577 if self.branchMap.has_key(repository): 578 branch = self.branchMap[repository] 579 elif self.branchMap.has_key('lp:' + repository): 580 branch = self.branchMap['lp:' + repository] 581 if not branch: 582 if self.defaultBranch: 583 branch = self.defaultBranch 584 else: 585 if repository: 586 branch = 'lp:' + repository 587 else: 588 branch = None 589 590 #log.msg("parse(): rev=%s who=%s files=%s comments='%s' when=%s branch=%s" % (rev, who, d['files'], d['comments'], time.asctime(time.localtime(when)), branch)) 591 if rev and who: 592 return changes.Change(who, d['files'], d['comments'], 593 when=when, revision=rev, branch=branch, 594 repository=repository or '') 595 else: 596 return None 597

598 -def parseLaunchpadDate(datestr, tz_sign, tz_hours, tz_minutes):

599 time_no_tz = calendar.timegm(time.strptime(datestr, "%Y-%m-%d %H:%M:%S")) 600 tz_delta = 60 * 60 * int(tz_sign + tz_hours) + 60 * int(tz_minutes) 601 return time_no_tz - tz_delta

602

Source Code for Module buildbot.changes.mail