1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 """
17 Parse various kinds of 'CVS notify' email.
18 """
19 import re
20 import time, calendar
21 import datetime
22 from email import message_from_file
23 from email.Utils import parseaddr, parsedate_tz, mktime_tz
24 from email.Iterators import body_line_iterator
25
26 from zope.interface import implements
27 from twisted.python import log
28 from twisted.internet import defer
29 from buildbot import util
30 from buildbot.interfaces import IChangeSource
31 from buildbot.util.maildir import MaildirService
32
34 """Generic base class for Maildir-based change sources"""
35 implements(IChangeSource)
36
37 compare_attrs = ["basedir", "pollinterval", "prefix"]
38
39 - def __init__(self, maildir, prefix=None, category='', repository=''):
47
49 return "%s watching maildir '%s'" % (self.__class__.__name__, self.basedir)
50
56 d.addCallback(parse_file)
57
58 def add_change(chdict):
59 if chdict:
60 return self.master.addChange(**chdict)
61 else:
62 log.msg("no change found in maildir file '%s'" % filename)
63 d.addCallback(add_change)
64
65 return d
66
68 m = message_from_file(fd)
69 return self.parse(m, prefix)
70
72 name = "CVSMaildirSource"
73
74 - def __init__(self, maildir, prefix=None, category='',
75 repository='', urlmaker=None, properties={}):
82
83 - def parse(self, m, prefix=None):
84 """Parse messages sent by the 'buildbot-cvs-mail' program.
85 """
86
87
88
89
90 name, addr = parseaddr(m["from"])
91 if not addr:
92 return None
93 at = addr.find("@")
94 if at == -1:
95 who = addr
96 else:
97 who = addr[:at]
98
99
100
101
102
103
104
105
106 log.msg('Processing CVS mail')
107 dateTuple = parsedate_tz(m["date"])
108 if dateTuple == None:
109 when = util.now()
110 else:
111 when = mktime_tz(dateTuple)
112
113 theTime = datetime.datetime.utcfromtimestamp(float(when))
114 rev = theTime.strftime('%Y-%m-%d %H:%M:%S')
115
116 catRE = re.compile( '^Category:\s*(\S.*)')
117 cvsRE = re.compile( '^CVSROOT:\s*(\S.*)')
118 cvsmodeRE = re.compile( '^Cvsmode:\s*(\S.*)')
119 filesRE = re.compile( '^Files:\s*(\S.*)')
120 modRE = re.compile( '^Module:\s*(\S.*)')
121 pathRE = re.compile( '^Path:\s*(\S.*)')
122 projRE = re.compile( '^Project:\s*(\S.*)')
123 singleFileRE = re.compile( '(.*) (NONE|\d(\.|\d)+) (NONE|\d(\.|\d)+)')
124 tagRE = re.compile( '^\s+Tag:\s*(\S.*)')
125 updateRE = re.compile( '^Update of:\s*(\S.*)')
126 comments = ""
127 branch = None
128 cvsroot = None
129 fileList = None
130 files = []
131 isdir = 0
132 path = None
133 project = None
134
135 lines = list(body_line_iterator(m))
136 while lines:
137 line = lines.pop(0)
138 m = catRE.match(line)
139 if m:
140 category = m.group(1)
141 continue
142 m = cvsRE.match(line)
143 if m:
144 cvsroot = m.group(1)
145 continue
146 m = cvsmodeRE.match(line)
147 if m:
148 cvsmode = m.group(1)
149 continue
150 m = filesRE.match(line)
151 if m:
152 fileList = m.group(1)
153 continue
154 m = modRE.match(line)
155 if m:
156
157
158 continue
159 m = pathRE.match(line)
160 if m:
161 path = m.group(1)
162 continue
163 m = projRE.match(line)
164 if m:
165 project = m.group(1)
166 continue
167 m = tagRE.match(line)
168 if m:
169 branch = m.group(1)
170 continue
171 m = updateRE.match(line)
172 if m:
173
174
175 continue
176 if line == "Log Message:\n":
177 break
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200 if fileList is None:
201 log.msg('CVSMaildirSource Mail with no files. Ignoring')
202 return None
203
204 if cvsmode == '1.11':
205
206 m = re.search('([^ ]*) ', fileList)
207 if m:
208 path = m.group(1)
209 else:
210 log.msg('CVSMaildirSource can\'t get path from file list. Ignoring mail')
211 return
212 fileList = fileList[len(path):].strip()
213 singleFileRE = re.compile( '(.+?),(NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+)),(NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+))(?: |$)')
214 elif cvsmode == '1.12':
215 singleFileRE = re.compile( '(.+?) (NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+)) (NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+))(?: |$)')
216 if path is None:
217 raise ValueError('CVSMaildirSource cvs 1.12 require path. Check cvs loginfo config')
218 else:
219 raise ValueError('Expected cvsmode 1.11 or 1.12. got: %s' % cvsmode)
220
221 log.msg("CVSMaildirSource processing filelist: %s" % fileList)
222 links = []
223 while(fileList):
224 m = singleFileRE.match(fileList)
225 if m:
226 curFile = path + '/' + m.group(1)
227 oldRev = m.group(2)
228 newRev = m.group(3)
229 files.append( curFile )
230 if self.urlmaker:
231 links.append(self.urlmaker(curFile, oldRev, newRev ))
232 fileList = fileList[m.end():]
233 else:
234 log.msg('CVSMaildirSource no files matched regex. Ignoring')
235 return None
236
237 while lines:
238 line = lines.pop(0)
239 comments += line
240
241 comments = comments.rstrip() + "\n"
242 if comments == '\n':
243 comments = None
244 return dict(
245 who=who,
246 files=files,
247 comments=comments,
248 isdir=isdir,
249 when=when,
250 branch=branch,
251 revision=rev,
252 category=category,
253 repository=cvsroot,
254 project=project,
255 links=links,
256 properties=self.properties)
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
284 name = "SVN commit-email.pl"
285
286 - def parse(self, m, prefix=None):
287 """Parse messages sent by the svn 'commit-email.pl' trigger.
288 """
289
290
291
292
293
294 name, addr = parseaddr(m["from"])
295 if not addr:
296 return None
297 at = addr.find("@")
298 if at == -1:
299 who = addr
300 else:
301 who = addr[:at]
302
303
304
305
306
307
308
309
310
311 when = util.now()
312
313 files = []
314 comments = ""
315 lines = list(body_line_iterator(m))
316 rev = None
317 while lines:
318 line = lines.pop(0)
319
320
321 match = re.search(r"^Author: (\S+)", line)
322 if match:
323 who = match.group(1)
324
325
326 match = re.search(r"^New Revision: (\d+)", line)
327 if match:
328 rev = match.group(1)
329
330
331
332
333
334
335
336
337 if (line == "Log:\n"):
338 break
339
340
341 while lines:
342 line = lines.pop(0)
343 if (line == "Modified:\n" or
344 line == "Added:\n" or
345 line == "Removed:\n"):
346 break
347 comments += line
348 comments = comments.rstrip() + "\n"
349
350 while lines:
351 line = lines.pop(0)
352 if line == "\n":
353 break
354 if line.find("Modified:\n") == 0:
355 continue
356 if line.find("Added:\n") == 0:
357 continue
358 if line.find("Removed:\n") == 0:
359 continue
360 line = line.strip()
361
362 thesefiles = line.split(" ")
363 for f in thesefiles:
364 if prefix:
365
366
367 if f.startswith(prefix):
368 f = f[len(prefix):]
369 else:
370 log.msg("ignored file from svn commit: prefix '%s' "
371 "does not match filename '%s'" % (prefix, f))
372 continue
373
374
375
376 files.append(f)
377
378 if not files:
379 log.msg("no matching files found, ignoring commit")
380 return None
381
382 return dict(
383 who=who,
384 files=files,
385 comments=comments,
386 when=when,
387 revision=rev)
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
417 name = "Launchpad"
418
419 compare_attrs = MaildirSource.compare_attrs + ["branchMap", "defaultBranch"]
420
421 - def __init__(self, maildir, prefix=None, branchMap=None, defaultBranch=None, **kwargs):
425
426 - def parse(self, m, prefix=None):
427 """Parse branch notification messages sent by Launchpad.
428 """
429
430 subject = m["subject"]
431 match = re.search(r"^\s*\[Branch\s+([^]]+)\]", subject)
432 if match:
433 repository = match.group(1)
434 else:
435 repository = None
436
437
438
439 d = { 'files': [], 'comments': "" }
440 gobbler = None
441 rev = None
442 who = None
443 when = util.now()
444 def gobble_comment(s):
445 d['comments'] += s + "\n"
446 def gobble_removed(s):
447 d['files'].append('%s REMOVED' % s)
448 def gobble_added(s):
449 d['files'].append('%s ADDED' % s)
450 def gobble_modified(s):
451 d['files'].append('%s MODIFIED' % s)
452 def gobble_renamed(s):
453 match = re.search(r"^(.+) => (.+)$", s)
454 if match:
455 d['files'].append('%s RENAMED %s' % (match.group(1), match.group(2)))
456 else:
457 d['files'].append('%s RENAMED' % s)
458
459 lines = list(body_line_iterator(m, True))
460 rev = None
461 while lines:
462 line = lines.pop(0)
463
464
465 match = re.search(r"^revno: ([0-9.]+)", line)
466 if match:
467 rev = match.group(1)
468
469
470 match = re.search(r"^committer: (.*)$", line)
471 if match:
472 who = match.group(1)
473
474
475
476
477 match = re.search(r"^timestamp: [a-zA-Z]{3} (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) ([-+])(\d{2})(\d{2})$", line)
478 if match:
479 datestr = match.group(1)
480 tz_sign = match.group(2)
481 tz_hours = match.group(3)
482 tz_minutes = match.group(4)
483 when = parseLaunchpadDate(datestr, tz_sign, tz_hours, tz_minutes)
484
485 if re.search(r"^message:\s*$", line):
486 gobbler = gobble_comment
487 elif re.search(r"^removed:\s*$", line):
488 gobbler = gobble_removed
489 elif re.search(r"^added:\s*$", line):
490 gobbler = gobble_added
491 elif re.search(r"^renamed:\s*$", line):
492 gobbler = gobble_renamed
493 elif re.search(r"^modified:\s*$", line):
494 gobbler = gobble_modified
495 elif re.search(r"^ ", line) and gobbler:
496 gobbler(line[2:-1])
497
498
499 branch = None
500 if self.branchMap and repository:
501 if self.branchMap.has_key(repository):
502 branch = self.branchMap[repository]
503 elif self.branchMap.has_key('lp:' + repository):
504 branch = self.branchMap['lp:' + repository]
505 if not branch:
506 if self.defaultBranch:
507 branch = self.defaultBranch
508 else:
509 if repository:
510 branch = 'lp:' + repository
511 else:
512 branch = None
513
514 if rev and who:
515 return dict(
516 who=who,
517 files=d['files'],
518 comments=d['comments'],
519 when=when,
520 revision=rev,
521 branch=branch,
522 repository=repository or '')
523 else:
524 return None
525
527 time_no_tz = calendar.timegm(time.strptime(datestr, "%Y-%m-%d %H:%M:%S"))
528 tz_delta = 60 * 60 * int(tz_sign + tz_hours) + 60 * int(tz_minutes)
529 return time_no_tz - tz_delta
530