buildbot.changes.mail

1 # -*- test-case-name: buildbot.test.test_mailparse -*- 2 3 """ 4 Parse various kinds of 'CVS notify' email. 5 """ 6 import os, re 7 import time, calendar 8 import datetime 9 from email import message_from_file 10 from email.Utils import parseaddr 11 from email.Iterators import body_line_iterator 12 13 from zope.interface import implements 14 from twisted.python import log 15 from buildbot import util 16 from buildbot.interfaces import IChangeSource 17 from buildbot.changes import changes 18 from buildbot.changes.maildir import MaildirService 19

20 -class MaildirSource(MaildirService, util.ComparableMixin):

21 """This source will watch a maildir that is subscribed to a FreshCVS 22 change-announcement mailing list. 23 """ 24 implements(IChangeSource) 25 26 compare_attrs = ["basedir", "pollinterval", "prefix"] 27 name = None 28

29 - def __init__(self, maildir, prefix=None, category='', repository=''):

30 MaildirService.__init__(self, maildir) 31 self.prefix = prefix 32 self.category = category 33 self.repository = repository 34 if prefix and not prefix.endswith("/"): 35 log.msg("%s: you probably want your prefix=('%s') to end with " 36 "a slash")

37

38 - def describe(self):

39 return "%s mailing list in maildir %s" % (self.name, self.basedir)

40

41 - def messageReceived(self, filename):

42 path = os.path.join(self.basedir, "new", filename) 43 change = self.parse_file(open(path, "r"), self.prefix) 44 if change: 45 self.parent.addChange(change) 46 os.rename(os.path.join(self.basedir, "new", filename), 47 os.path.join(self.basedir, "cur", filename))

48

49 - def parse_file(self, fd, prefix=None):

50 m = message_from_file(fd) 51 return self.parse(m, prefix)

52

53 -class FCMaildirSource(MaildirSource):

54 name = "FreshCVS" 55

56 - def parse(self, m, prefix=None):

57 """Parse mail sent by FreshCVS""" 58 59 # FreshCVS sets From: to "user CVS <user>", but the <> part may be 60 # modified by the MTA (to include a local domain) 61 name, addr = parseaddr(m["from"]) 62 if not name: 63 return None # no From means this message isn't from FreshCVS 64 cvs = name.find(" CVS") 65 if cvs == -1: 66 return None # this message isn't from FreshCVS 67 who = name[:cvs] 68 69 # we take the time of receipt as the time of checkin. Not correct, 70 # but it avoids the out-of-order-changes issue. See the comment in 71 # parseSyncmail about using the 'Date:' header 72 when = util.now() 73 74 files = [] 75 comments = "" 76 isdir = 0 77 lines = list(body_line_iterator(m)) 78 while lines: 79 line = lines.pop(0) 80 if line == "Modified files:\n": 81 break 82 while lines: 83 line = lines.pop(0) 84 if line == "\n": 85 break 86 line = line.rstrip("\n") 87 linebits = line.split(None, 1) 88 file = linebits[0] 89 if prefix: 90 # insist that the file start with the prefix: FreshCVS sends 91 # changes we don't care about too 92 if file.startswith(prefix): 93 file = file[len(prefix):] 94 else: 95 continue 96 if len(linebits) == 1: 97 isdir = 1 98 elif linebits[1] == "0 0": 99 isdir = 1 100 files.append(file) 101 while lines: 102 line = lines.pop(0) 103 if line == "Log message:\n": 104 break 105 # message is terminated by "ViewCVS links:" or "Index:..." (patch) 106 while lines: 107 line = lines.pop(0) 108 if line == "ViewCVS links:\n": 109 break 110 if line.find("Index: ") == 0: 111 break 112 comments += line 113 comments = comments.rstrip() + "\n" 114 115 if not files: 116 return None 117 118 change = changes.Change(who, files, comments, isdir, when=when) 119 120 return change

121

122 -class SyncmailMaildirSource(MaildirSource):

123 name = "Syncmail" 124

125 - def parse(self, m, prefix=None):

126 """Parse messages sent by the 'syncmail' program, as suggested by the 127 sourceforge.net CVS Admin documentation. Syncmail is maintained at 128 syncmail.sf.net . 129 """ 130 # pretty much the same as freshcvs mail, not surprising since CVS is 131 # the one creating most of the text 132 133 # The mail is sent from the person doing the checkin. Assume that the 134 # local username is enough to identify them (this assumes a one-server 135 # cvs-over-rsh environment rather than the server-dirs-shared-over-NFS 136 # model) 137 name, addr = parseaddr(m["from"]) 138 if not addr: 139 return None # no From means this message isn't from FreshCVS 140 at = addr.find("@") 141 if at == -1: 142 who = addr # might still be useful 143 else: 144 who = addr[:at] 145 146 # we take the time of receipt as the time of checkin. Not correct (it 147 # depends upon the email latency), but it avoids the 148 # out-of-order-changes issue. Also syncmail doesn't give us anything 149 # better to work with, unless you count pulling the v1-vs-v2 150 # timestamp out of the diffs, which would be ugly. TODO: Pulling the 151 # 'Date:' header from the mail is a possibility, and 152 # email.Utils.parsedate_tz may be useful. It should be configurable, 153 # however, because there are a lot of broken clocks out there. 154 when = util.now() 155 156 # calculate a "revision" based on that timestamp 157 theCurrentTime = datetime.datetime.utcfromtimestamp(float(when)) 158 rev = theCurrentTime.strftime('%Y-%m-%d %H:%M:%S') 159 160 subject = m["subject"] 161 # syncmail puts the repository-relative directory in the subject: 162 # mprefix + "%(dir)s %(file)s,%(oldversion)s,%(newversion)s", where 163 # 'mprefix' is something that could be added by a mailing list 164 # manager. 165 # this is the only reasonable way to determine the directory name 166 space = subject.find(" ") 167 if space != -1: 168 directory = subject[:space] 169 else: 170 directory = subject 171 172 files = [] 173 comments = "" 174 isdir = 0 175 branch = None 176 177 lines = list(body_line_iterator(m)) 178 while lines: 179 line = lines.pop(0) 180 181 if (line == "Modified Files:\n" or 182 line == "Added Files:\n" or 183 line == "Removed Files:\n"): 184 break 185 186 while lines: 187 line = lines.pop(0) 188 if line == "\n": 189 break 190 if line == "Log Message:\n": 191 lines.insert(0, line) 192 break 193 line = line.lstrip() 194 line = line.rstrip() 195 # note: syncmail will send one email per directory involved in a 196 # commit, with multiple files if they were in the same directory. 197 # Unlike freshCVS, it makes no attempt to collect all related 198 # commits into a single message. 199 200 # note: syncmail will report a Tag underneath the ... Files: line 201 # e.g.: Tag: BRANCH-DEVEL 202 203 if line.startswith('Tag:'): 204 branch = line.split(' ')[-1].rstrip() 205 continue 206 207 thesefiles = line.split(" ") 208 for f in thesefiles: 209 f = directory + "/" + f 210 if prefix: 211 # insist that the file start with the prefix: we may get 212 # changes we don't care about too 213 if f.startswith(prefix): 214 f = f[len(prefix):] 215 else: 216 continue 217 break 218 # TODO: figure out how new directories are described, set 219 # .isdir 220 files.append(f) 221 222 if not files: 223 return None 224 225 while lines: 226 line = lines.pop(0) 227 if line == "Log Message:\n": 228 break 229 # message is terminated by "Index:..." (patch) or "--- NEW FILE.." 230 # or "--- filename DELETED ---". Sigh. 231 while lines: 232 line = lines.pop(0) 233 if line.find("Index: ") == 0: 234 break 235 if re.search(r"^--- NEW FILE", line): 236 break 237 if re.search(r" DELETED ---$", line): 238 break 239 comments += line 240 comments = comments.rstrip() + "\n" 241 242 change = changes.Change(who, files, comments, isdir, when=when, 243 branch=branch, revision=rev, 244 category=self.category, 245 repository=self.repository) 246 247 return change

248 249 # Bonsai mail parser by Stephen Davis. 250 # 251 # This handles changes for CVS repositories that are watched by Bonsai 252 # (http://www.mozilla.org/bonsai.html) 253 254 # A Bonsai-formatted email message looks like: 255 # 256 # C|1071099907|stephend|/cvs|Sources/Scripts/buildbot|bonsai.py|1.2|||18|7 257 # A|1071099907|stephend|/cvs|Sources/Scripts/buildbot|master.cfg|1.1|||18|7 258 # R|1071099907|stephend|/cvs|Sources/Scripts/buildbot|BuildMaster.py||| 259 # LOGCOMMENT 260 # Updated bonsai parser and switched master config to buildbot-0.4.1 style. 261 # 262 # :ENDLOGCOMMENT 263 # 264 # In the first example line, stephend is the user, /cvs the repository, 265 # buildbot the directory, bonsai.py the file, 1.2 the revision, no sticky 266 # and branch, 18 lines added and 7 removed. All of these fields might not be 267 # present (during "removes" for example). 268 # 269 # There may be multiple "control" lines or even none (imports, directory 270 # additions) but there is one email per directory. We only care about actual 271 # changes since it is presumed directory additions don't actually affect the 272 # build. At least one file should need to change (the makefile, say) to 273 # actually make a new directory part of the build process. That's my story 274 # and I'm sticking to it. 275

276 -class BonsaiMaildirSource(MaildirSource):

277 name = "Bonsai" 278

279 - def parse(self, m, prefix=None):

280 """Parse mail sent by the Bonsai cvs loginfo script.""" 281 282 # we don't care who the email came from b/c the cvs user is in the 283 # msg text 284 285 who = "unknown" 286 timestamp = None 287 files = [] 288 lines = list(body_line_iterator(m)) 289 290 # read the control lines (what/who/where/file/etc.) 291 while lines: 292 line = lines.pop(0) 293 if line == "LOGCOMMENT\n": 294 break; 295 line = line.rstrip("\n") 296 297 # we'd like to do the following but it won't work if the number of 298 # items doesn't match so... 299 # what, timestamp, user, repo, module, file = line.split( '|' ) 300 items = line.split('|') 301 if len(items) < 6: 302 # not a valid line, assume this isn't a bonsai message 303 return None 304 305 try: 306 # just grab the bottom-most timestamp, they're probably all the 307 # same. TODO: I'm assuming this is relative to the epoch, but 308 # this needs testing. 309 timestamp = int(items[1]) 310 except ValueError: 311 pass 312 313 user = items[2] 314 if user: 315 who = user 316 317 module = items[4] 318 file = items[5] 319 if module and file: 320 path = "%s/%s" % (module, file) 321 files.append(path) 322 sticky = items[7] 323 branch = items[8] 324 325 # if no files changed, return nothing 326 if not files: 327 return None 328 329 # read the comments 330 comments = "" 331 while lines: 332 line = lines.pop(0) 333 if line == ":ENDLOGCOMMENT\n": 334 break 335 comments += line 336 comments = comments.rstrip() + "\n" 337 338 # return buildbot Change object 339 return changes.Change(who, files, comments, when=timestamp, 340 branch=branch)

341 342 # svn "commit-email.pl" handler. The format is very similar to freshcvs mail; 343 # here's a sample: 344 345 # From: username [at] apache.org [slightly obfuscated to avoid spam here] 346 # To: commits [at] spamassassin.apache.org 347 # Subject: svn commit: r105955 - in spamassassin/trunk: . lib/Mail 348 # ... 349 # 350 # Author: username 351 # Date: Sat Nov 20 00:17:49 2004 [note: TZ = local tz on server!] 352 # New Revision: 105955 353 # 354 # Modified: [also Removed: and Added:] 355 # [filename] 356 # ... 357 # Log: 358 # [log message] 359 # ... 360 # 361 # 362 # Modified: spamassassin/trunk/lib/Mail/SpamAssassin.pm 363 # [unified diff] 364 # 365 # [end of mail] 366

367 -class SVNCommitEmailMaildirSource(MaildirSource):

368 name = "SVN commit-email.pl" 369

370 - def parse(self, m, prefix=None):

371 """Parse messages sent by the svn 'commit-email.pl' trigger. 372 """ 373 374 # The mail is sent from the person doing the checkin. Assume that the 375 # local username is enough to identify them (this assumes a one-server 376 # cvs-over-rsh environment rather than the server-dirs-shared-over-NFS 377 # model) 378 name, addr = parseaddr(m["from"]) 379 if not addr: 380 return None # no From means this message isn't from FreshCVS 381 at = addr.find("@") 382 if at == -1: 383 who = addr # might still be useful 384 else: 385 who = addr[:at] 386 387 # we take the time of receipt as the time of checkin. Not correct (it 388 # depends upon the email latency), but it avoids the 389 # out-of-order-changes issue. Also syncmail doesn't give us anything 390 # better to work with, unless you count pulling the v1-vs-v2 391 # timestamp out of the diffs, which would be ugly. TODO: Pulling the 392 # 'Date:' header from the mail is a possibility, and 393 # email.Utils.parsedate_tz may be useful. It should be configurable, 394 # however, because there are a lot of broken clocks out there. 395 when = util.now() 396 397 files = [] 398 comments = "" 399 isdir = 0 400 lines = list(body_line_iterator(m)) 401 rev = None 402 while lines: 403 line = lines.pop(0) 404 405 # "Author: jmason" 406 match = re.search(r"^Author: (\S+)", line) 407 if match: 408 who = match.group(1) 409 410 # "New Revision: 105955" 411 match = re.search(r"^New Revision: (\d+)", line) 412 if match: 413 rev = match.group(1) 414 415 # possible TODO: use "Date: ..." data here instead of time of 416 # commit message receipt, above. however, this timestamp is 417 # specified *without* a timezone, in the server's local TZ, so to 418 # be accurate buildbot would need a config setting to specify the 419 # source server's expected TZ setting! messy. 420 421 # this stanza ends with the "Log:" 422 if (line == "Log:\n"): 423 break 424 425 # commit message is terminated by the file-listing section 426 while lines: 427 line = lines.pop(0) 428 if (line == "Modified:\n" or 429 line == "Added:\n" or 430 line == "Removed:\n"): 431 break 432 comments += line 433 comments = comments.rstrip() + "\n" 434 435 while lines: 436 line = lines.pop(0) 437 if line == "\n": 438 break 439 if line.find("Modified:\n") == 0: 440 continue # ignore this line 441 if line.find("Added:\n") == 0: 442 continue # ignore this line 443 if line.find("Removed:\n") == 0: 444 continue # ignore this line 445 line = line.strip() 446 447 thesefiles = line.split(" ") 448 for f in thesefiles: 449 if prefix: 450 # insist that the file start with the prefix: we may get 451 # changes we don't care about too 452 if f.startswith(prefix): 453 f = f[len(prefix):] 454 else: 455 log.msg("ignored file from svn commit: prefix '%s' " 456 "does not match filename '%s'" % (prefix, f)) 457 continue 458 459 # TODO: figure out how new directories are described, set 460 # .isdir 461 files.append(f) 462 463 if not files: 464 log.msg("no matching files found, ignoring commit") 465 return None 466 467 return changes.Change(who, files, comments, when=when, revision=rev)

468 469 # bzr Launchpad branch subscription mails. Sample mail: 470 # 471 # From: noreply@launchpad.net 472 # Subject: [Branch ~knielsen/maria/tmp-buildbot-test] Rev 2701: test add file 473 # To: Joe <joe@acme.com> 474 # ... 475 # 476 # ------------------------------------------------------------ 477 # revno: 2701 478 # committer: Joe <joe@acme.com> 479 # branch nick: tmpbb 480 # timestamp: Fri 2009-05-15 10:35:43 +0200 481 # message: 482 # test add file 483 # added: 484 # test-add-file 485 # 486 # 487 # -- 488 # 489 # https://code.launchpad.net/~knielsen/maria/tmp-buildbot-test 490 # 491 # You are subscribed to branch lp:~knielsen/maria/tmp-buildbot-test. 492 # To unsubscribe from this branch go to https://code.launchpad.net/~knielsen/maria/tmp-buildbot-test/+edit-subscription. 493 # 494 # [end of mail] 495

496 -class BzrLaunchpadEmailMaildirSource(MaildirSource):

497 name = "Launchpad" 498 499 compare_attrs = MaildirSource.compare_attrs + ["branchMap", "defaultBranch"] 500

501 - def __init__(self, maildir, prefix=None, branchMap=None, defaultBranch=None, **kwargs):

502 self.branchMap = branchMap 503 self.defaultBranch = defaultBranch 504 MaildirSource.__init__(self, maildir, prefix, **kwargs)

505

506 - def parse(self, m, prefix=None):

507 """Parse branch notification messages sent by Launchpad. 508 """ 509 510 subject = m["subject"] 511 match = re.search(r"^\s*\[Branch\s+([^]]+)\]", subject) 512 if match: 513 repository = match.group(1) 514 else: 515 repository = None 516 517 # Put these into a dictionary, otherwise we cannot assign them 518 # from nested function definitions. 519 d = { 'files': [], 'comments': "" } 520 gobbler = None 521 rev = None 522 who = None 523 when = util.now() 524 def gobble_comment(s): 525 d['comments'] += s + "\n"

526 def gobble_removed(s): 527 d['files'].append('%s REMOVED' % s)

528 def gobble_added(s): 529 d['files'].append('%s ADDED' % s) 530 def gobble_modified(s): 531 d['files'].append('%s MODIFIED' % s) 532 def gobble_renamed(s): 533 match = re.search(r"^(.+) => (.+)$", s) 534 if match: 535 d['files'].append('%s RENAMED %s' % (match.group(1), match.group(2))) 536 else: 537 d['files'].append('%s RENAMED' % s) 538 539 lines = list(body_line_iterator(m, True)) 540 rev = None 541 while lines: 542 line = lines.pop(0) 543 544 # revno: 101 545 match = re.search(r"^revno: ([0-9.]+)", line) 546 if match: 547 rev = match.group(1) 548 549 # committer: Joe <joe@acme.com> 550 match = re.search(r"^committer: (.*)$", line) 551 if match: 552 who = match.group(1) 553 554 # timestamp: Fri 2009-05-15 10:35:43 +0200 555 # datetime.strptime() is supposed to support %z for time zone, but 556 # it does not seem to work. So handle the time zone manually. 557 match = re.search(r"^timestamp: [a-zA-Z]{3} (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) ([-+])(\d{2})(\d{2})$", line) 558 if match: 559 datestr = match.group(1) 560 tz_sign = match.group(2) 561 tz_hours = match.group(3) 562 tz_minutes = match.group(4) 563 when = parseLaunchpadDate(datestr, tz_sign, tz_hours, tz_minutes) 564 565 if re.search(r"^message:\s*$", line): 566 gobbler = gobble_comment 567 elif re.search(r"^removed:\s*$", line): 568 gobbler = gobble_removed 569 elif re.search(r"^added:\s*$", line): 570 gobbler = gobble_added 571 elif re.search(r"^renamed:\s*$", line): 572 gobbler = gobble_renamed 573 elif re.search(r"^modified:\s*$", line): 574 gobbler = gobble_modified 575 elif re.search(r"^ ", line) and gobbler: 576 gobbler(line[2:-1]) # Use :-1 to gobble trailing newline 577 578 # Determine the name of the branch. 579 branch = None 580 if self.branchMap and repository: 581 if self.branchMap.has_key(repository): 582 branch = self.branchMap[repository] 583 elif self.branchMap.has_key('lp:' + repository): 584 branch = self.branchMap['lp:' + repository] 585 if not branch: 586 if self.defaultBranch: 587 branch = self.defaultBranch 588 else: 589 if repository: 590 branch = 'lp:' + repository 591 else: 592 branch = None 593 594 #log.msg("parse(): rev=%s who=%s files=%s comments='%s' when=%s branch=%s" % (rev, who, d['files'], d['comments'], time.asctime(time.localtime(when)), branch)) 595 if rev and who: 596 return changes.Change(who, d['files'], d['comments'], 597 when=when, revision=rev, branch=branch, 598 repository=repository or '') 599 else: 600 return None 601

602 -def parseLaunchpadDate(datestr, tz_sign, tz_hours, tz_minutes):

603 time_no_tz = calendar.timegm(time.strptime(datestr, "%Y-%m-%d %H:%M:%S")) 604 tz_delta = 60 * 60 * int(tz_sign + tz_hours) + 60 * int(tz_minutes) 605 return time_no_tz - tz_delta

606

Source Code for Module buildbot.changes.mail