1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 """
18 Parse various kinds of 'CVS notify' email.
19 """
20 import os, re
21 import time, calendar
22 import datetime
23 from email import message_from_file
24 from email.Utils import parseaddr, parsedate_tz, mktime_tz
25 from email.Iterators import body_line_iterator
26
27 from zope.interface import implements
28 from twisted.python import log
29 from buildbot import util
30 from buildbot.interfaces import IChangeSource
31 from buildbot.changes import changes
32 from buildbot.changes.maildir import MaildirService
33
35 """This source will watch a maildir that is subscribed to a FreshCVS
36 change-announcement mailing list.
37 """
38 implements(IChangeSource)
39
40 compare_attrs = ["basedir", "pollinterval", "prefix"]
41 name = None
42
43 - def __init__(self, maildir, prefix=None, category='', repository=''):
51
53 return "%s mailing list in maildir %s" % (self.name, self.basedir)
54
62
64 m = message_from_file(fd)
65 return self.parse(m, prefix)
66
68 name = "CVSMaildirSource"
69
70 - def __init__(self, maildir, prefix=None, category='',
71 repository='', urlmaker=None, properties={}):
78
79 - def parse(self, m, prefix=None):
80 """Parse messages sent by the 'buildbot-cvs-mail' program.
81 """
82
83
84
85
86 name, addr = parseaddr(m["from"])
87 if not addr:
88 return None
89 at = addr.find("@")
90 if at == -1:
91 who = addr
92 else:
93 who = addr[:at]
94
95
96
97
98
99
100
101
102 log.msg('Processing CVS mail')
103 dateTuple = parsedate_tz(m["date"])
104 if dateTuple == None:
105 when = util.now()
106 else:
107 when = mktime_tz(dateTuple)
108
109 theTime = datetime.datetime.utcfromtimestamp(float(when))
110 rev = theTime.strftime('%Y-%m-%d %H:%M:%S')
111
112 catRE = re.compile( '^Category:\s*(\S.*)')
113 cvsRE = re.compile( '^CVSROOT:\s*(\S.*)')
114 cvsmodeRE = re.compile( '^Cvsmode:\s*(\S.*)')
115 filesRE = re.compile( '^Files:\s*(\S.*)')
116 modRE = re.compile( '^Module:\s*(\S.*)')
117 pathRE = re.compile( '^Path:\s*(\S.*)')
118 projRE = re.compile( '^Project:\s*(\S.*)')
119 singleFileRE = re.compile( '(.*) (NONE|\d(\.|\d)+) (NONE|\d(\.|\d)+)')
120 tagRE = re.compile( '^\s+Tag:\s*(\S.*)')
121 updateRE = re.compile( '^Update of:\s*(\S.*)')
122 comments = ""
123 branch = None
124 cvsroot = None
125 fileList = None
126 files = []
127 isdir = 0
128 path = None
129 project = None
130
131 lines = list(body_line_iterator(m))
132 while lines:
133 line = lines.pop(0)
134 m = catRE.match(line)
135 if m:
136 category = m.group(1)
137 continue
138 m = cvsRE.match(line)
139 if m:
140 cvsroot = m.group(1)
141 continue
142 m = cvsmodeRE.match(line)
143 if m:
144 cvsmode = m.group(1)
145 continue
146 m = filesRE.match(line)
147 if m:
148 fileList = m.group(1)
149 continue
150 m = modRE.match(line)
151 if m:
152
153
154 continue
155 m = pathRE.match(line)
156 if m:
157 path = m.group(1)
158 continue
159 m = projRE.match(line)
160 if m:
161 project = m.group(1)
162 continue
163 m = tagRE.match(line)
164 if m:
165 branch = m.group(1)
166 continue
167 m = updateRE.match(line)
168 if m:
169
170
171 continue
172 if line == "Log Message:\n":
173 break
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196 if fileList is None:
197 log.msg('CVSMaildirSource Mail with no files. Ignoring')
198 return None
199
200 if cvsmode == '1.11':
201
202 m = re.search('([^ ]*) ', fileList)
203 if m:
204 path = m.group(1)
205 else:
206 log.msg('CVSMaildirSource can\'t get path from file list. Ignoring mail')
207 return
208 fileList = fileList[len(path):].strip()
209 singleFileRE = re.compile( '(.+?),(NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+)),(NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+))(?: |$)')
210 elif cvsmode == '1.12':
211 singleFileRE = re.compile( '(.+?) (NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+)) (NONE|(?:\d+\.(?:\d+\.\d+\.)*\d+))(?: |$)')
212 if path is None:
213 raise ValueError('CVSMaildirSource cvs 1.12 require path. Check cvs loginfo config')
214 else:
215 raise ValueError('Expected cvsmode 1.11 or 1.12. got: %s' % cvsmode)
216
217 log.msg("CVSMaildirSource processing filelist: %s" % fileList)
218 links = []
219 while(fileList):
220 m = singleFileRE.match(fileList)
221 if m:
222 curFile = path + '/' + m.group(1)
223 oldRev = m.group(2)
224 newRev = m.group(3)
225 files.append( curFile )
226 if self.urlmaker:
227 links.append(self.urlmaker(curFile, oldRev, newRev ))
228 fileList = fileList[m.end():]
229 else:
230 log.msg('CVSMaildirSource no files matched regex. Ignoring')
231 return None
232
233 while lines:
234 line = lines.pop(0)
235 comments += line
236
237 comments = comments.rstrip() + "\n"
238 if comments == '\n':
239 comments = None
240 change = changes.Change(who, files, comments, isdir, when=when,
241 branch=branch, revision=rev,
242 category=category,
243 repository=cvsroot,
244 project=project,
245 links=links,
246 properties=self.properties)
247 return change
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
275 name = "SVN commit-email.pl"
276
277 - def parse(self, m, prefix=None):
278 """Parse messages sent by the svn 'commit-email.pl' trigger.
279 """
280
281
282
283
284
285 name, addr = parseaddr(m["from"])
286 if not addr:
287 return None
288 at = addr.find("@")
289 if at == -1:
290 who = addr
291 else:
292 who = addr[:at]
293
294
295
296
297
298
299
300
301
302 when = util.now()
303
304 files = []
305 comments = ""
306 lines = list(body_line_iterator(m))
307 rev = None
308 while lines:
309 line = lines.pop(0)
310
311
312 match = re.search(r"^Author: (\S+)", line)
313 if match:
314 who = match.group(1)
315
316
317 match = re.search(r"^New Revision: (\d+)", line)
318 if match:
319 rev = match.group(1)
320
321
322
323
324
325
326
327
328 if (line == "Log:\n"):
329 break
330
331
332 while lines:
333 line = lines.pop(0)
334 if (line == "Modified:\n" or
335 line == "Added:\n" or
336 line == "Removed:\n"):
337 break
338 comments += line
339 comments = comments.rstrip() + "\n"
340
341 while lines:
342 line = lines.pop(0)
343 if line == "\n":
344 break
345 if line.find("Modified:\n") == 0:
346 continue
347 if line.find("Added:\n") == 0:
348 continue
349 if line.find("Removed:\n") == 0:
350 continue
351 line = line.strip()
352
353 thesefiles = line.split(" ")
354 for f in thesefiles:
355 if prefix:
356
357
358 if f.startswith(prefix):
359 f = f[len(prefix):]
360 else:
361 log.msg("ignored file from svn commit: prefix '%s' "
362 "does not match filename '%s'" % (prefix, f))
363 continue
364
365
366
367 files.append(f)
368
369 if not files:
370 log.msg("no matching files found, ignoring commit")
371 return None
372
373 return changes.Change(who, files, comments, when=when, revision=rev)
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
403 name = "Launchpad"
404
405 compare_attrs = MaildirSource.compare_attrs + ["branchMap", "defaultBranch"]
406
407 - def __init__(self, maildir, prefix=None, branchMap=None, defaultBranch=None, **kwargs):
411
412 - def parse(self, m, prefix=None):
413 """Parse branch notification messages sent by Launchpad.
414 """
415
416 subject = m["subject"]
417 match = re.search(r"^\s*\[Branch\s+([^]]+)\]", subject)
418 if match:
419 repository = match.group(1)
420 else:
421 repository = None
422
423
424
425 d = { 'files': [], 'comments': "" }
426 gobbler = None
427 rev = None
428 who = None
429 when = util.now()
430 def gobble_comment(s):
431 d['comments'] += s + "\n"
432 def gobble_removed(s):
433 d['files'].append('%s REMOVED' % s)
434 def gobble_added(s):
435 d['files'].append('%s ADDED' % s)
436 def gobble_modified(s):
437 d['files'].append('%s MODIFIED' % s)
438 def gobble_renamed(s):
439 match = re.search(r"^(.+) => (.+)$", s)
440 if match:
441 d['files'].append('%s RENAMED %s' % (match.group(1), match.group(2)))
442 else:
443 d['files'].append('%s RENAMED' % s)
444
445 lines = list(body_line_iterator(m, True))
446 rev = None
447 while lines:
448 line = lines.pop(0)
449
450
451 match = re.search(r"^revno: ([0-9.]+)", line)
452 if match:
453 rev = match.group(1)
454
455
456 match = re.search(r"^committer: (.*)$", line)
457 if match:
458 who = match.group(1)
459
460
461
462
463 match = re.search(r"^timestamp: [a-zA-Z]{3} (\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) ([-+])(\d{2})(\d{2})$", line)
464 if match:
465 datestr = match.group(1)
466 tz_sign = match.group(2)
467 tz_hours = match.group(3)
468 tz_minutes = match.group(4)
469 when = parseLaunchpadDate(datestr, tz_sign, tz_hours, tz_minutes)
470
471 if re.search(r"^message:\s*$", line):
472 gobbler = gobble_comment
473 elif re.search(r"^removed:\s*$", line):
474 gobbler = gobble_removed
475 elif re.search(r"^added:\s*$", line):
476 gobbler = gobble_added
477 elif re.search(r"^renamed:\s*$", line):
478 gobbler = gobble_renamed
479 elif re.search(r"^modified:\s*$", line):
480 gobbler = gobble_modified
481 elif re.search(r"^ ", line) and gobbler:
482 gobbler(line[2:-1])
483
484
485 branch = None
486 if self.branchMap and repository:
487 if self.branchMap.has_key(repository):
488 branch = self.branchMap[repository]
489 elif self.branchMap.has_key('lp:' + repository):
490 branch = self.branchMap['lp:' + repository]
491 if not branch:
492 if self.defaultBranch:
493 branch = self.defaultBranch
494 else:
495 if repository:
496 branch = 'lp:' + repository
497 else:
498 branch = None
499
500
501 if rev and who:
502 return changes.Change(who, d['files'], d['comments'],
503 when=when, revision=rev, branch=branch,
504 repository=repository or '')
505 else:
506 return None
507
509 time_no_tz = calendar.timegm(time.strptime(datestr, "%Y-%m-%d %H:%M:%S"))
510 tz_delta = 60 * 60 * int(tz_sign + tz_hours) + 60 * int(tz_minutes)
511 return time_no_tz - tz_delta
512