Source Code for Module buildbot.util.maildir

  1  # This file is part of Buildbot.  Buildbot is free software: you can 
  2  # redistribute it and/or modify it under the terms of the GNU General Public 
  3  # License as published by the Free Software Foundation, version 2. 
  4  # 
  5  # This program is distributed in the hope that it will be useful, but WITHOUT 
  6  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
  7  # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more 
  8  # details. 
  9  # 
 10  # You should have received a copy of the GNU General Public License along with 
 11  # this program; if not, write to the Free Software Foundation, Inc., 51 
 12  # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 
 13  # 
 14  # Copyright Buildbot Team Members 
 15   
 16   
 17  # This is a class which watches a maildir for new messages. It uses the 
 18  # linux dirwatcher API (if available) to look for new files. The 
 19  # .messageReceived method is invoked with the filename of the new message, 
 20  # relative to the top of the maildir (so it will look like "new/blahblah"). 
 21   
 22  import os 
 23  from twisted.python import log, runtime 
 24  from twisted.application import service, internet 
 25  from twisted.internet import reactor, defer 
 26  dnotify = None 
 27  try: 
 28      import dnotify 
 29  except: 
 30      log.msg("unable to import dnotify, so Maildir will use polling instead") 
 31   
 32 -class NoSuchMaildir(Exception): 
 33      pass 
 34   
 35 -class MaildirService(service.MultiService): 
 36      """I watch a maildir for new messages. I should be placed as the service 
 37      child of some MultiService instance. When running, I use the linux 
 38      dirwatcher API (if available) or poll for new files in the 'new' 
 39      subdirectory of my maildir path. When I discover a new message, I invoke 
 40      my .messageReceived() method with the short filename of the new message, 
 41      so the full name of the new file can be obtained with 
 42      os.path.join(maildir, 'new', filename). messageReceived() should be 
 43      overridden by a subclass to do something useful. I will not move or 
 44      delete the file on my own: the subclass's messageReceived() should 
 45      probably do that. 
 46      """ 
 47      pollinterval = 10  # only used if we don't have DNotify 
 48   
 49 -    def __init__(self, basedir=None): 
 50          """Create the Maildir watcher. BASEDIR is the maildir directory (the 
 51          one which contains new/ and tmp/) 
 52          """ 
 53          service.MultiService.__init__(self) 
 54          if basedir: 
 55              self.setBasedir(basedir) 
 56          self.files = [] 
 57          self.dnotify = None 
 58   
 59 -    def setBasedir(self, basedir): 
 60          # some users of MaildirService (scheduler.Try_Jobdir, in particular) 
 61          # don't know their basedir until setServiceParent, since it is 
 62          # relative to the buildmaster's basedir. So let them set it late. We 
 63          # don't actually need it until our own startService. 
 64          self.basedir = basedir 
 65          self.newdir = os.path.join(self.basedir, "new") 
 66          self.curdir = os.path.join(self.basedir, "cur") 
 67   
 68 -    def startService(self): 
 69          service.MultiService.startService(self) 
 70          if not os.path.isdir(self.newdir) or not os.path.isdir(self.curdir): 
 71              raise NoSuchMaildir("invalid maildir '%s'" % self.basedir) 
 72          try: 
 73              if dnotify: 
 74                  # we must hold an fd open on the directory, so we can get 
 75                  # notified when it changes. 
 76                  self.dnotify = dnotify.DNotify(self.newdir, 
 77                                                 self.dnotify_callback, 
 78                                                 [dnotify.DNotify.DN_CREATE]) 
 79          except (IOError, OverflowError): 
 80              # IOError is probably linux<2.4.19, which doesn't support 
 81              # dnotify. OverflowError will occur on some 64-bit machines 
 82              # because of a python bug 
 83              log.msg("DNotify failed, falling back to polling") 
 84          if not self.dnotify: 
 85              t = internet.TimerService(self.pollinterval, self.poll) 
 86              t.setServiceParent(self) 
 87          self.poll() 
 88   
 89 -    def dnotify_callback(self): 
 90          log.msg("dnotify noticed something, now polling") 
 91   
 92          # give it a moment. I found that qmail had problems when the message 
 93          # was removed from the maildir instantly. It shouldn't, that's what 
 94          # maildirs are made for. I wasn't able to eyeball any reason for the 
 95          # problem, and safecat didn't behave the same way, but qmail reports 
 96          # "Temporary_error_on_maildir_delivery" (qmail-local.c:165, 
 97          # maildir_child() process exited with rc not in 0,2,3,4). Not sure 
 98          # why, and I'd have to hack qmail to investigate further, so it's 
 99          # easier to just wait a second before yanking the message out of new/ 
100   
101          reactor.callLater(0.1, self.poll) 
102   
103   
104 -    def stopService(self): 
105          if self.dnotify: 
106              self.dnotify.remove() 
107              self.dnotify = None 
108          return service.MultiService.stopService(self) 
109   
110      @defer.deferredGenerator 
111 -    def poll(self): 
112          assert self.basedir 
113          # see what's new 
114          for f in self.files: 
115              if not os.path.isfile(os.path.join(self.newdir, f)): 
116                  self.files.remove(f) 
117          newfiles = [] 
118          for f in os.listdir(self.newdir): 
119              if not f in self.files: 
120                  newfiles.append(f) 
121          self.files.extend(newfiles) 
122          for n in newfiles: 
123              try: 
124                  wfd = defer.waitForDeferred(self.messageReceived(n)) 
125                  yield wfd 
126                  wfd.getResult() 
127              except: 
128                  log.msg("while reading '%s' from maildir '%s':" % (n, self.basedir)) 
129                  log.err() 
130   
131 -    def moveToCurDir(self, filename): 
132          """ 
133          Call this from messageReceived to start processing the message; this 
134          moves the message file to the 'cur' directory and returns an open file 
135          handle for it. 
136   
137          @param filename: unqualified filename of the message 
138          @returns: open file 
139          """ 
140          if runtime.platformType == "posix": 
141              # open the file before moving it, because I'm afraid that once 
142              # it's in cur/, someone might delete it at any moment 
143              path = os.path.join(self.newdir, filename) 
144              f = open(path, "r") 
145              os.rename(os.path.join(self.newdir, filename), 
146                        os.path.join(self.curdir, filename)) 
147          elif runtime.platformType == "win32": 
148              # do this backwards under windows, because you can't move a file 
149              # that somebody is holding open. This was causing a Permission 
150              # Denied error on bear's win32-twisted1.3 buildslave. 
151              os.rename(os.path.join(self.newdir, filename), 
152                        os.path.join(self.curdir, filename)) 
153              path = os.path.join(self.curdir, filename) 
154              f = open(path, "r") 
155   
156          return f 
157   
158 -    def messageReceived(self, filename): 
159          """Process a received message.  The filename is relative to self.newdir. 
160          Returns a Deferred.""" 
161          raise NotImplementedError 
162