Package buildbot :: Package db :: Module enginestrategy
[frames] | no frames]

Source Code for Module buildbot.db.enginestrategy

  1  # This file is part of Buildbot.  Buildbot is free software: you can 
  2  # redistribute it and/or modify it under the terms of the GNU General Public 
  3  # License as published by the Free Software Foundation, version 2. 
  4  # 
  5  # This program is distributed in the hope that it will be useful, but WITHOUT 
  6  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
  7  # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more 
  8  # details. 
  9  # 
 10  # You should have received a copy of the GNU General Public License along with 
 11  # this program; if not, write to the Free Software Foundation, Inc., 51 
 12  # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 
 13  # 
 14  # Copyright Buildbot Team Members 
 15   
 16  """ 
 17  A wrapper around `sqlalchemy.create_engine` that handles all of the 
 18  special cases that Buildbot needs.  Those include: 
 19   
 20   - pool_recycle for MySQL 
 21   - %(basedir) substitution 
 22   - optimal thread pool size calculation 
 23   
 24  """ 
 25   
 26  import os 
 27  import sqlalchemy as sa 
 28  from twisted.python import log 
 29  from sqlalchemy.engine import strategies, url 
 30  from sqlalchemy.pool import NullPool, Pool 
 31  from buildbot.util import sautils 
 32   
 33  # from http://www.mail-archive.com/sqlalchemy@googlegroups.com/msg15079.html 
34 -class ReconnectingListener(object):
35 - def __init__(self):
36 self.retried = False
37
38 -class BuildbotEngineStrategy(strategies.ThreadLocalEngineStrategy):
39 # A subclass of the ThreadLocalEngineStrategy that can effectively interact 40 # with Buildbot. 41 # 42 # This adjusts the passed-in parameters to ensure that we get the behaviors 43 # Buildbot wants from particular drivers, and wraps the outgoing Engine 44 # object so that its methods run in threads and return deferreds. 45 46 name = 'buildbot' 47
48 - def special_case_sqlite(self, u, kwargs):
49 """For sqlite, percent-substitute %(basedir)s and use a full 50 path to the basedir. If using a memory database, force the 51 pool size to be 1.""" 52 max_conns = None 53 54 # when given a database path, stick the basedir in there 55 if u.database: 56 57 # Use NullPool instead of the sqlalchemy-0.6.8-default 58 # SingletonThreadpool for sqlite to suppress the error in 59 # http://groups.google.com/group/sqlalchemy/msg/f8482e4721a89589, 60 # which also explains that NullPool is the new default in 61 # sqlalchemy 0.7 for non-memory SQLite databases. 62 kwargs.setdefault('poolclass', NullPool) 63 64 u.database = u.database % dict(basedir = kwargs['basedir']) 65 if not os.path.isabs(u.database[0]): 66 u.database = os.path.join(kwargs['basedir'], u.database) 67 68 # in-memory databases need exactly one connection 69 if not u.database: 70 kwargs['pool_size'] = 1 71 max_conns = 1 72 73 # allow serializing access to the db 74 if 'serialize_access' in u.query: 75 u.query.pop('serialize_access') 76 max_conns = 1 77 78 return u, kwargs, max_conns
79
80 - def set_up_sqlite_engine(self, u, engine):
81 """Special setup for sqlite engines""" 82 # try to enable WAL logging 83 if u.database: 84 log.msg("setting database journal mode to 'wal'") 85 try: 86 engine.execute("pragma journal_mode = wal") 87 except: 88 log.msg("failed to set journal mode - database may fail")
89
90 - def special_case_mysql(self, u, kwargs):
91 """For mysql, take max_idle out of the query arguments, and 92 use its value for pool_recycle. Also, force use_unicode and 93 charset to be True and 'utf8', failing if they were set to 94 anything else.""" 95 96 kwargs['pool_recycle'] = int(u.query.pop('max_idle', 3600)) 97 98 # default to the InnoDB storage engine 99 storage_engine = u.query.pop('storage_engine', 'MyISAM') 100 kwargs['connect_args'] = { 101 'init_command' : 'SET storage_engine=%s' % storage_engine, 102 } 103 104 if 'use_unicode' in u.query: 105 if u.query['use_unicode'] != "True": 106 raise TypeError("Buildbot requires use_unicode=True " + 107 "(and adds it automatically)") 108 else: 109 u.query['use_unicode'] = True 110 111 if 'charset' in u.query: 112 if u.query['charset'] != "utf8": 113 raise TypeError("Buildbot requires charset=utf8 " + 114 "(and adds it automatically)") 115 else: 116 u.query['charset'] = 'utf8' 117 118 # add the reconnecting PoolListener that will detect a 119 # disconnected connection and automatically start a new 120 # one. This provides a measure of additional safety over 121 # the pool_recycle parameter, and is useful when e.g., the 122 # mysql server goes away 123 def checkout_listener(dbapi_con, con_record, con_proxy): 124 try: 125 cursor = dbapi_con.cursor() 126 cursor.execute("SELECT 1") 127 except dbapi_con.OperationalError, ex: 128 if ex.args[0] in (2006, 2013, 2014, 2045, 2055): 129 # sqlalchemy will re-create the connection 130 raise sa.exc.DisconnectionError() 131 raise
132 133 # older versions of sqlalchemy require the listener to be specified 134 # in the kwargs, in a class instance 135 if sautils.sa_version() < (0,7,0): 136 class ReconnectingListener(object): 137 pass
138 rcl = ReconnectingListener() 139 rcl.checkout = checkout_listener 140 kwargs['listeners'] = [ rcl ] 141 else: 142 sa.event.listen(Pool, 'checkout', checkout_listener) 143 144 return u, kwargs, None 145
146 - def create(self, name_or_url, **kwargs):
147 if 'basedir' not in kwargs: 148 raise TypeError('no basedir supplied to create_engine') 149 150 max_conns = None 151 152 # apply special cases 153 u = url.make_url(name_or_url) 154 if u.drivername.startswith('sqlite'): 155 u, kwargs, max_conns = self.special_case_sqlite(u, kwargs) 156 elif u.drivername.startswith('mysql'): 157 u, kwargs, max_conns = self.special_case_mysql(u, kwargs) 158 159 # remove the basedir as it may confuse sqlalchemy 160 basedir = kwargs.pop('basedir') 161 162 # calculate the maximum number of connections from the pool parameters, 163 # if it hasn't already been specified 164 if max_conns is None: 165 max_conns = kwargs.get('pool_size', 5) + kwargs.get('max_overflow', 10) 166 167 engine = strategies.ThreadLocalEngineStrategy.create(self, 168 u, **kwargs) 169 170 # annotate the engine with the optimal thread pool size; this is used 171 # by DBConnector to configure the surrounding thread pool 172 engine.optimal_thread_pool_size = max_conns 173 174 # keep the basedir 175 engine.buildbot_basedir = basedir 176 177 if u.drivername.startswith('sqlite'): 178 self.set_up_sqlite_engine(u, engine) 179 180 return engine
181 182 BuildbotEngineStrategy() 183 184 # this module is really imported for the side-effects, but pyflakes will like 185 # us to use something from the module -- so offer a copy of create_engine, 186 # which explicitly adds the strategy argument
187 -def create_engine(*args, **kwargs):
188 kwargs['strategy'] = 'buildbot' 189 190 return sa.create_engine(*args, **kwargs)
191