buildbot.changes.mail

1 # This file is part of Buildbot. Buildbot is free software: you can 2 # redistribute it and/or modify it under the terms of the GNU General Public 3 # License as published by the Free Software Foundation, version 2. 4 # 5 # This program is distributed in the hope that it will be useful, but WITHOUT 6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 7 # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 8 # details. 9 # 10 # You should have received a copy of the GNU General Public License along with 11 # this program; if not, write to the Free Software Foundation, Inc., 51 12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 13 # 14 # Copyright Buildbot Team Members 15 16 """ 17 Parse various kinds of 'CVS notify' email. 18 """ 19 import re 20 import time, calendar 21 import datetime 22 from email import message_from_file 23 from email.Utils import parseaddr, parsedate_tz, mktime_tz 24 from email.Iterators import body_line_iterator 25 26 from zope.interface import implements 27 from twisted.python import log 28 from twisted.internet import defer 29 from buildbot import util 30 from buildbot.interfaces import IChangeSource 31 from buildbot.util.maildir import MaildirService 32

33 -class MaildirSource(MaildirService, util.ComparableMixin):

34 """Generic base class for Maildir-based change sources""" 35 implements(IChangeSource) 36 37 compare_attrs = ["basedir", "pollinterval", "prefix"] 38

39 - def __init__(self, maildir, prefix=None, category='', repository=''):

40 MaildirService.__init__(self, maildir) 41 self.prefix = prefix 42 self.category = category 43 self.repository = repository 44 if prefix and not prefix.endswith("/"): 45 log.msg("%s: you probably want your prefix=('%s') to end with " 46 "a slash")

47

48 - def describe(self):

49 return "%s watching maildir '%s'" % (self.__class__.__name__, self.basedir)

50

51 - def messageReceived(self, filename):

52 d = defer.succeed(None) 53 def parse_file(_): 54 f = self.moveToCurDir(filename) 55 return self.parse_file(f, self.prefix)

56 d.addCallback(parse_file) 57 58 def add_change(chdict): 59 if chdict: 60 return self.master.addChange(**chdict) 61 else: 62 log.msg("no change found in maildir file '%s'" % filename)

63 d.addCallback(add_change) 64 65 return d 66

67 - def parse_file(self, fd, prefix=None):

68 m = message_from_file(fd) 69 return self.parse(m, prefix)

70

71 -class CVSMaildirSource(MaildirSource):

72 name = "CVSMaildirSource" 73

74 - def __init__(self, maildir, prefix=None, category='', 75 repository='', urlmaker=None, properties={}):

76 """If urlmaker is defined, it will be called with three arguments: 77 filename, previous version, new version. It returns a url for that 78 file.""" 79 MaildirSource.__init__(self, maildir, prefix, category, repository) 80 self.urlmaker = urlmaker 81 self.properties = properties

82

83 - def parse(self, m, prefix=None):

84 """Parse messages sent by the 'buildbot-cvs-mail' program. 85 """ 86 # The mail is sent from the person doing the checkin. Assume that the 87 # local username is enough to identify them (this assumes a one-server 88 # cvs-over-rsh environment rather than the server-dirs-shared-over-NFS 89 # model) 90 name, addr = parseaddr(m["from"]) 91 if not addr: 92 return None # no From means this message isn't from buildbot-cvs-mail 93 at = addr.find("@") 94 if at == -1: 95 who = addr # might still be useful 96 else: 97 who = addr[:at] 98 99 # CVS accecpts RFC822 dates. buildbot-cvs-mail adds the date as 100 # part of the mail header, so use that. 101 # This assumes cvs is being access via ssh or pserver, so the time 102 # will be the CVS server's time. 103 104 # calculate a "revision" based on that timestamp, or the current time 105 # if we're unable to parse the date. 106 log.msg('Processing CVS mail') 107 dateTuple = parsedate_tz(m["date"]) 108 if dateTuple == None: 109 when = util.now() 110 else: 111 when = mktime_tz(dateTuple) 112 113 theTime = datetime.datetime.utcfromtimestamp(float(when)) 114 rev = theTime.strftime('%Y-%m-%d %H:%M:%S') 115 116 catRE = re.compile( '^Category:\s*(\S.*)') 117 cvsRE = re.compile( '^CVSROOT:\s*(\S.*)') 118 cvsmodeRE = re.compile( '^Cvsmode:\s*(\S.*)') 119 filesRE = re.compile( '^Files:\s*(\S.*)') 120 modRE = re.compile( '^Module:\s*(\S.*)') 121 pathRE = re.compile( '^Path:\s*(\S.*)') 122 projRE = re.compile( '^Project:\s*(\S.*)') 123 singleFileRE = re.compile( '(.*) (NONE|\d(\.|\d)+) (NONE|\d(\.|\d)+)') 124 tagRE = re.compile( '^\s+Tag:\s*(\S.*)') 125 updateRE = re.compile( '^Update of:\s*(\S.*)') 126 comments = "" 127 branch = None 128 cvsroot = None 129 fileList = None 130 files = [] 131 isdir = 0 132 path = None 133 project = None 134 135 lines = list(body_line_iterator(m)) 136 while lines: 137 line = lines.pop(0) 138 m = catRE.match(line) 139 if m: 140 category = m.group(1) 141 continue 142 m = cvsRE.match(line) 143 if m: 144 cvsroot = m.group(1) 145 continue 146 m = cvsmodeRE.match(line) 147 if m: 148 cvsmode = m.group(1) 149 continue 150 m = filesRE.match(line) 151 if m: 152 fileList = m.group(1) 153 continue 154 m = modRE.match(line) 155 if m: 156 # We don't actually use this 157 #module = m.group(1) 158 continue 159 m = pathRE.match(line) 160 if m: 161 path = m.group(1) 162 continue 163 m = projRE.match(line) 164 if m: 165 project = m.group(1) 166 continue 167 m = tagRE.match(line) 168 if m: 169 branch = m.group(1) 170 continue 171 m = updateRE.match(line) 172 if m: 173 # We don't actually use this 174 #updateof = m.group(1) 175 continue 176 if line == "Log Message:\n": 177 break 178 179 # CVS 1.11 lists files as: 180 # repo/path file,old-version,new-version file2,old-version,new-version 181 # Version 1.12 lists files as: 182 # file1 old-version new-version file2 old-version new-version 183 # 184 # files consists of tuples of 'file-name old-version new-version' 185 # The versions are either dotted-decimal version numbers, ie 1.1 186 # or NONE. New files are of the form 'NONE NUMBER', while removed 187 # files are 'NUMBER NONE'. 'NONE' is a literal string 188 # Parsing this instead of files list in 'Added File:' etc 189 # makes it possible to handle files with embedded spaces, though 190 # it could fail if the filename was 'bad 1.1 1.2' 191 # For cvs version 1.11, we expect 192 # my_module new_file.c,NONE,1.1 193 # my_module removed.txt,1.2,NONE 194 # my_module modified_file.c,1.1,1.2 195 # While cvs version 1.12 gives us 196 # new_file.c NONE 1.1 197 # removed.txt 1.2 NONE 198 # modified_file.c 1.1,1.2 199 200 if fileList is None: 201 log.msg('CVSMaildirSource Mail with no files. Ignoring') 202 return None # We don't have any files. Email not from CVS 203 204 if cvsmode == '1.11': 205 # Please, no repo paths with spaces! 206 m = re.search('([^ ]*) ', fileList) 207 if m: 208 path = m.group(1) 209 else: 210 log.msg('CVSMaildirSource can\'t get path from file list. Ignoring mail') 211 return 212 fileList = fileList[len(path):].strip() 213 singleFileRE = re.compile( '(.+?),(NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+)),(NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+))(?: |$)') 214 elif cvsmode == '1.12': 215 singleFileRE = re.compile( '(.+?) (NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+)) (NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+))(?: |$)') 216 if path is None: 217 raise ValueError('CVSMaildirSource cvs 1.12 require path. Check cvs loginfo config') 218 else: 219 raise ValueError('Expected cvsmode 1.11 or 1.12. got: %s' % cvsmode) 220 221 log.msg("CVSMaildirSource processing filelist: %s" % fileList) 222 links = [] 223 while(fileList): 224 m = singleFileRE.match(fileList) 225 if m: 226 curFile = path + '/' + m.group(1) 227 oldRev = m.group(2) 228 newRev = m.group(3) 229 files.append( curFile ) 230 if self.urlmaker: 231 links.append(self.urlmaker(curFile, oldRev, newRev )) 232 fileList = fileList[m.end():] 233 else: 234 log.msg('CVSMaildirSource no files matched regex. Ignoring') 235 return None # bail - we couldn't parse the files that changed 236 # Now get comments 237 while lines: 238 line = lines.pop(0) 239 comments += line 240 241 comments = comments.rstrip() + "\n" 242 if comments == '\n': 243 comments = None 244 return dict( 245 who=who, 246 files=files, 247 comments=comments, 248 isdir=isdir, 249 when=when, 250 branch=branch, 251 revision=rev, 252 category=category, 253 repository=cvsroot, 254 project=project, 255 links=links, 256 properties=self.properties)

257 258 # svn "commit-email.pl" handler. The format is very similar to freshcvs mail; 259 # here's a sample: 260 261 # From: username [at] apache.org [slightly obfuscated to avoid spam here] 262 # To: commits [at] spamassassin.apache.org 263 # Subject: svn commit: r105955 - in spamassassin/trunk: . lib/Mail 264 # ... 265 # 266 # Author: username 267 # Date: Sat Nov 20 00:17:49 2004 [note: TZ = local tz on server!] 268 # New Revision: 105955 269 # 270 # Modified: [also Removed: and Added:] 271 # [filename] 272 # ... 273 # Log: 274 # [log message] 275 # ... 276 # 277 # 278 # Modified: spamassassin/trunk/lib/Mail/SpamAssassin.pm 279 # [unified diff] 280 # 281 # [end of mail] 282

283 -class SVNCommitEmailMaildirSource(MaildirSource):

284 name = "SVN commit-email.pl" 285

286 - def parse(self, m, prefix=None):

287 """Parse messages sent by the svn 'commit-email.pl' trigger. 288 """ 289 290 # The mail is sent from the person doing the checkin. Assume that the 291 # local username is enough to identify them (this assumes a one-server 292 # cvs-over-rsh environment rather than the server-dirs-shared-over-NFS 293 # model) 294 name, addr = parseaddr(m["from"]) 295 if not addr: 296 return None # no From means this message isn't from svn 297 at = addr.find("@") 298 if at == -1: 299 who = addr # might still be useful 300 else: 301 who = addr[:at] 302 303 # we take the time of receipt as the time of checkin. Not correct (it 304 # depends upon the email latency), but it avoids the 305 # out-of-order-changes issue. Also syncmail doesn't give us anything 306 # better to work with, unless you count pulling the v1-vs-v2 307 # timestamp out of the diffs, which would be ugly. TODO: Pulling the 308 # 'Date:' header from the mail is a possibility, and 309 # email.Utils.parsedate_tz may be useful. It should be configurable, 310 # however, because there are a lot of broken clocks out there. 311 when = util.now() 312 313 files = [] 314 comments = "" 315 lines = list(body_line_iterator(m)) 316 rev = None 317 while lines: 318 line = lines.pop(0) 319 320 # "Author: jmason" 321 match = re.search(r"^Author: (\S+)", line) 322 if match: 323 who = match.group(1) 324 325 # "New Revision: 105955" 326 match = re.search(r"^New Revision: (\d+)", line) 327 if match: 328 rev = match.group(1) 329 330 # possible TODO: use "Date: ..." data here instead of time of 331 # commit message receipt, above. however, this timestamp is 332 # specified *without* a timezone, in the server's local TZ, so to 333 # be accurate buildbot would need a config setting to specify the 334 # source server's expected TZ setting! messy. 335 336 # this stanza ends with the "Log:" 337 if (line == "Log:\n"): 338 break 339 340 # commit message is terminated by the file-listing section 341 while lines: 342 line = lines.pop(0) 343 if (line == "Modified:\n" or 344 line == "Added:\n" or 345 line == "Removed:\n"): 346 break 347 comments += line 348 comments = comments.rstrip() + "\n" 349 350 while lines: 351 line = lines.pop(0) 352 if line == "\n": 353 break 354 if line.find("Modified:\n") == 0: 355 continue # ignore this line 356 if line.find("Added:\n") == 0: 357 continue # ignore this line 358 if line.find("Removed:\n") == 0: 359 continue # ignore this line 360 line = line.strip() 361 362 thesefiles = line.split(" ") 363 for f in thesefiles: 364 if prefix: 365 # insist that the file start with the prefix: we may get 366 # changes we don't care about too 367 if f.startswith(prefix): 368 f = f[len(prefix):] 369 else: 370 log.msg("ignored file from svn commit: prefix '%s' " 371 "does not match filename '%s'" % (prefix, f)) 372 continue 373 374 # TODO: figure out how new directories are described, set 375 # .isdir 376 files.append(f) 377 378 if not files: 379 log.msg("no matching files found, ignoring commit") 380 return None 381 382 return dict( 383 who=who, 384 files=files, 385 comments=comments, 386 when=when, 387 revision=rev)

388 389 # bzr Launchpad branch subscription mails. Sample mail: 390 # 391 # From: noreply@launchpad.net 392 # Subject: [Branch ~knielsen/maria/tmp-buildbot-test] Rev 2701: test add file 393 # To: Joe <joe@acme.com> 394 # ... 395 # 396 # ------------------------------------------------------------ 397 # revno: 2701 398 # committer: Joe <joe@acme.com> 399 # branch nick: tmpbb 400 # timestamp: Fri 2009-05-15 10:35:43 +0200 401 # message: 402 # test add file 403 # added: 404 # test-add-file 405 # 406 # 407 # -- 408 # 409 # https://code.launchpad.net/~knielsen/maria/tmp-buildbot-test 410 # 411 # You are subscribed to branch lp:~knielsen/maria/tmp-buildbot-test. 412 # To unsubscribe from this branch go to https://code.launchpad.net/~knielsen/maria/tmp-buildbot-test/+edit-subscription. 413 # 414 # [end of mail] 415

416 -class BzrLaunchpadEmailMaildirSource(MaildirSource):

417 name = "Launchpad" 418 419 compare_attrs = MaildirSource.compare_attrs + ["branchMap", "defaultBranch"] 420

421 - def __init__(self, maildir, prefix=None, branchMap=None, defaultBranch=None, **kwargs):

422 self.branchMap = branchMap 423 self.defaultBranch = defaultBranch 424 MaildirSource.__init__(self, maildir, prefix, **kwargs)

425

426 - def parse(self, m, prefix=None):

427 """Parse branch notification messages sent by Launchpad. 428 """ 429 430 subject = m["subject"] 431 match = re.search(r"^\s*\[Branch\s+([^]]+)\]", subject) 432 if match: 433 repository = match.group(1) 434 else: 435 repository = None 436 437 # Put these into a dictionary, otherwise we cannot assign them 438 # from nested function definitions. 439 d = { 'files': [], 'comments': "" } 440 gobbler = None 441 rev = None 442 who = None 443 when = util.now() 444 def gobble_comment(s): 445 d['comments'] += s + "\n"

446 def gobble_removed(s): 447 d['files'].append('%s REMOVED' % s)

448 def gobble_added(s): 449 d['files'].append('%s ADDED' % s) 450 def gobble_modified(s): 451 d['files'].append('%s MODIFIED' % s) 452 def gobble_renamed(s): 453 match = re.search(r"^(.+) => (.+)$", s) 454 if match: 455 d['files'].append('%s RENAMED %s' % (match.group(1), match.group(2))) 456 else: 457 d['files'].append('%s RENAMED' % s) 458 459 lines = list(body_line_iterator(m, True)) 460 rev = None 461 while lines: 462 line = lines.pop(0) 463 464 # revno: 101 465 match = re.search(r"^revno: ([0-9.]+)", line) 466 if match: 467 rev = match.group(1) 468 469 # committer: Joe <joe@acme.com> 470 match = re.search(r"^committer: (.*)$", line) 471 if match: 472 who = match.group(1) 473 474 # timestamp: Fri 2009-05-15 10:35:43 +0200 475 # datetime.strptime() is supposed to support %z for time zone, but 476 # it does not seem to work. So handle the time zone manually. 477 match = re.search(r"^timestamp: [a-zA-Z]{3} (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) ([-+])(\d{2})(\d{2})$", line) 478 if match: 479 datestr = match.group(1) 480 tz_sign = match.group(2) 481 tz_hours = match.group(3) 482 tz_minutes = match.group(4) 483 when = parseLaunchpadDate(datestr, tz_sign, tz_hours, tz_minutes) 484 485 if re.search(r"^message:\s*$", line): 486 gobbler = gobble_comment 487 elif re.search(r"^removed:\s*$", line): 488 gobbler = gobble_removed 489 elif re.search(r"^added:\s*$", line): 490 gobbler = gobble_added 491 elif re.search(r"^renamed:\s*$", line): 492 gobbler = gobble_renamed 493 elif re.search(r"^modified:\s*$", line): 494 gobbler = gobble_modified 495 elif re.search(r"^ ", line) and gobbler: 496 gobbler(line[2:-1]) # Use :-1 to gobble trailing newline 497 498 # Determine the name of the branch. 499 branch = None 500 if self.branchMap and repository: 501 if self.branchMap.has_key(repository): 502 branch = self.branchMap[repository] 503 elif self.branchMap.has_key('lp:' + repository): 504 branch = self.branchMap['lp:' + repository] 505 if not branch: 506 if self.defaultBranch: 507 branch = self.defaultBranch 508 else: 509 if repository: 510 branch = 'lp:' + repository 511 else: 512 branch = None 513 514 if rev and who: 515 return dict( 516 who=who, 517 files=d['files'], 518 comments=d['comments'], 519 when=when, 520 revision=rev, 521 branch=branch, 522 repository=repository or '') 523 else: 524 return None 525

526 -def parseLaunchpadDate(datestr, tz_sign, tz_hours, tz_minutes):

527 time_no_tz = calendar.timegm(time.strptime(datestr, "%Y-%m-%d %H:%M:%S")) 528 tz_delta = 60 * 60 * int(tz_sign + tz_hours) + 60 * int(tz_minutes) 529 return time_no_tz - tz_delta

530

Source Code for Module buildbot.changes.mail