Source Code for Module buildbot.db.enginestrategy

  1  # This file is part of Buildbot.  Buildbot is free software: you can 
  2  # redistribute it and/or modify it under the terms of the GNU General Public 
  3  # License as published by the Free Software Foundation, version 2. 
  4  # 
  5  # This program is distributed in the hope that it will be useful, but WITHOUT 
  6  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
  7  # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more 
  8  # details. 
  9  # 
 10  # You should have received a copy of the GNU General Public License along with 
 11  # this program; if not, write to the Free Software Foundation, Inc., 51 
 12  # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 
 13  # 
 14  # Copyright Buildbot Team Members 
 15   
 16  """ 
 17  A wrapper around `sqlalchemy.create_engine` that handles all of the 
 18  special cases that Buildbot needs.  Those include: 
 19   
 20   - pool_recycle for MySQL 
 21   - %(basedir) substitution 
 22   - optimal thread pool size calculation 
 23   
 24  """ 
 25   
 26  import os 
 27  import sqlalchemy 
 28  from sqlalchemy.engine import strategies, url 
 29  from sqlalchemy.pool import NullPool 
 30   
 31  # from http://www.mail-archive.com/sqlalchemy@googlegroups.com/msg15079.html 
 32 -class ReconnectingListener(object): 
 33 -    def __init__(self): 
 34          self.retried = False 
 35 -    def checkout(self, dbapi_con, con_record, con_proxy): 
 36          try: 
 37              try: 
 38                  dbapi_con.ping(False) 
 39              except TypeError: 
 40                  dbapi_con.ping() 
 41          except dbapi_con.OperationalError, ex: 
 42              if ex.args[0] in (2006, 2013, 2014, 2045, 2055): 
 43                  # sqlalchemy will re-create the connection 
 44                  raise sqlalchemy.exc.DisconnectionError() 
 45              raise 
 46   
 47 -class BuildbotEngineStrategy(strategies.ThreadLocalEngineStrategy): 
 48      """ 
 49      A subclass of the ThreadLocalEngineStrategy that can effectively interact 
 50      with Buildbot. 
 51   
 52      This adjusts the passed-in parameters to ensure that we get the behaviors 
 53      Buildbot wants from particular drivers, and wraps the outgoing Engine 
 54      object so that its methods run in threads and return deferreds. 
 55      """ 
 56   
 57      name = 'buildbot' 
 58   
 59 -    def special_case_sqlite(self, u, kwargs): 
 60          """For sqlite, percent-substitute %(basedir)s and use a full 
 61          path to the basedir.  If using a memory database, force the 
 62          pool size to be 1.""" 
 63          max_conns = None 
 64   
 65          # when given a database path, stick the basedir in there 
 66          if u.database: 
 67   
 68              # Use NullPool instead of the sqlalchemy-0.6.8-default 
 69              # SingletonThreadpool for sqlite to suppress the error in 
 70              # http://groups.google.com/group/sqlalchemy/msg/f8482e4721a89589, 
 71              # which also explains that NullPool is the new default in 
 72              # sqlalchemy 0.7 for non-memory SQLite databases. 
 73              kwargs.setdefault('poolclass', NullPool) 
 74   
 75              u.database = u.database % dict(basedir = kwargs['basedir']) 
 76              if not os.path.isabs(u.database[0]): 
 77                  u.database = os.path.join(kwargs['basedir'], u.database) 
 78   
 79          # in-memory databases need exactly one connection 
 80          if not u.database: 
 81              kwargs['pool_size'] = 1 
 82              max_conns = 1 
 83   
 84          # allow serializing access to the db 
 85          if 'serialize_access' in u.query: 
 86              u.query.pop('serialize_access') 
 87              max_conns = 1 
 88   
 89          return u, kwargs, max_conns 
 90   
 91 -    def special_case_mysql(self, u, kwargs): 
 92          """For mysql, take max_idle out of the query arguments, and 
 93          use its value for pool_recycle.  Also, force use_unicode and 
 94          charset to be True and 'utf8', failing if they were set to 
 95          anything else.""" 
 96   
 97          kwargs['pool_recycle'] = int(u.query.pop('max_idle', 3600)) 
 98   
 99          # default to the InnoDB storage engine 
100          storage_engine = u.query.pop('storage_engine', 'MyISAM') 
101          kwargs['connect_args'] = { 
102              'init_command' : 'SET storage_engine=%s' % storage_engine, 
103          } 
104   
105          if 'use_unicode' in u.query: 
106              if u.query['use_unicode'] != "True": 
107                  raise TypeError("Buildbot requires use_unicode=True " + 
108                                   "(and adds it automatically)") 
109          else: 
110              u.query['use_unicode'] = True 
111   
112          if 'charset' in u.query: 
113              if u.query['charset'] != "utf8": 
114                  raise TypeError("Buildbot requires charset=utf8 " + 
115                                   "(and adds it automatically)") 
116          else: 
117              u.query['charset'] = 'utf8' 
118   
119          # add the reconnecting PoolListener that will detect a 
120          # disconnected connection and automatically start a new 
121          # one.  This provides a measure of additional safety over 
122          # the pool_recycle parameter, and is useful when e.g., the 
123          # mysql server goes away 
124          kwargs['listeners'] = [ ReconnectingListener() ] 
125   
126          return u, kwargs, None 
127   
128 -    def create(self, name_or_url, **kwargs): 
129          if 'basedir' not in kwargs: 
130              raise TypeError('no basedir supplied to create_engine') 
131   
132          max_conns = None 
133   
134          # apply special cases 
135          u = url.make_url(name_or_url) 
136          if u.drivername.startswith('sqlite'): 
137              u, kwargs, max_conns = self.special_case_sqlite(u, kwargs) 
138          elif u.drivername.startswith('mysql'): 
139              u, kwargs, max_conns = self.special_case_mysql(u, kwargs) 
140   
141          # remove the basedir as it may confuse sqlalchemy 
142          basedir = kwargs.pop('basedir') 
143   
144          # calculate the maximum number of connections from the pool parameters, 
145          # if it hasn't already been specified 
146          if max_conns is None: 
147              max_conns = kwargs.get('pool_size', 5) + kwargs.get('max_overflow', 10) 
148   
149          engine = strategies.ThreadLocalEngineStrategy.create(self, 
150                                              u, **kwargs) 
151   
152          # annotate the engine with the optimal thread pool size; this is used 
153          # by DBConnector to configure the surrounding thread pool 
154          engine.optimal_thread_pool_size = max_conns 
155   
156          # and keep the basedir 
157          engine.buildbot_basedir = basedir 
158   
159          return engine 
160   
161  BuildbotEngineStrategy() 
162   
163  # this module is really imported for the side-effects, but pyflakes will like 
164  # us to use something from the module -- so offer a copy of create_engine, which 
165  # explicitly adds the strategy argument 
166 -def create_engine(*args, **kwargs): 
167      kwargs['strategy'] = 'buildbot' 
168   
169      return sqlalchemy.create_engine(*args, **kwargs) 
170