[multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
also, SqliteConnectionWrapper becomes a subclass. This is allowed since
now ConnectionsSet responsability has been reduced to handling the system
source only.
This changeset also:
* drops useless cnxset_set / check_connection api
* deprecates former .source / .connection / container api
but it renames neither the session's cnxset attribute, nor the related
repository method.
Related to #2919300
--- a/server/pool.py Thu Jun 27 09:18:39 2013 +0200
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,126 +0,0 @@
-# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
-# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
-#
-# This file is part of CubicWeb.
-#
-# CubicWeb is free software: you can redistribute it and/or modify it under the
-# terms of the GNU Lesser General Public License as published by the Free
-# Software Foundation, either version 2.1 of the License, or (at your option)
-# any later version.
-#
-# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
-# details.
-#
-# You should have received a copy of the GNU Lesser General Public License along
-# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
-"""CubicWeb server connections set : the repository has a limited number of
-:class:`ConnectionsSet` (defined in configuration, default to 4). Each of them
-hold a connection to the system source.
-"""
-
-__docformat__ = "restructuredtext en"
-
-import sys
-
-from logilab.common.deprecation import deprecated
-
-
-class ConnectionsSet(object):
- """handle connection to the system source, at some point associated to a
- :class:`Session`
- """
-
- # since 3.19, we only have to manage the system source connection
- def __init__(self, system_source):
- # dictionary of (source, connection), indexed by sources'uri
- self._source = system_source
- self.cnx = system_source.get_connection()
- self.cu = self.cnx.cursor()
-
- def commit(self):
- """commit the current transaction for this user"""
- # let exception propagates
- self.cnx.commit()
-
- def rollback(self):
- """rollback the current transaction for this user"""
- # catch exceptions, rollback other sources anyway
- try:
- self.cnx.rollback()
- except Exception:
- self._source.critical('rollback error', exc_info=sys.exc_info())
- # error on rollback, the connection is much probably in a really
- # bad state. Replace it by a new one.
- self.reconnect()
-
- def close(self, i_know_what_i_do=False):
- """close all connections in the set"""
- if i_know_what_i_do is not True: # unexpected closing safety belt
- raise RuntimeError('connections set shouldn\'t be closed')
- try:
- self.cu.close()
- self.cu = None
- except Exception:
- pass
- try:
- self.cnx.close()
- except Exception:
- pass
-
- # internals ###############################################################
-
- def cnxset_set(self):
- """connections set is being set on a session"""
- self.check_connections()
-
- def cnxset_freed(self):
- """connections set is being freed from a session"""
- pass # do nothing by default
-
- def reconnect(self):
- """reopen a connection for this source or all sources if none specified
- """
- try:
- # properly close existing connection if any
- self.cnx.close()
- except Exception:
- pass
- self._source.info('trying to reconnect')
- self.cnx = self._source.get_connection()
- self.cu = self.cnx.cursor()
-
- def check_connections(self):
- newcnx = self._source.check_connection(self.cnx)
- if newcnx is not None:
- self.cnx = newcnx
- self.cu = self.cnx.cursor()
-
- @deprecated('[3.19] use .cu instead')
- def __getitem__(self, uri):
- assert uri == 'system'
- return self.cu
-
- @deprecated('[3.19] use repo.system_source instead')
- def source(self, uid):
- assert uid == 'system'
- return self._source
-
- @deprecated('[3.19] use .cnx instead')
- def connection(self, uid):
- assert uid == 'system'
- return self.cnx
-
- @property
- @deprecated('[3.19] use .cnx instead')
- def source_cnxs(self):
- return {'system': (self._source, self.cnx)}
-
-
-from cubicweb.server.hook import Operation, LateOperation, SingleLastOperation
-from logilab.common.deprecation import class_moved, class_renamed
-Operation = class_moved(Operation)
-PreCommitOperation = class_renamed('PreCommitOperation', Operation)
-LateOperation = class_moved(LateOperation)
-SingleLastOperation = class_moved(SingleLastOperation)
--- a/server/repository.py Thu Jun 27 09:18:39 2013 +0200
+++ b/server/repository.py Wed Jan 22 15:50:03 2014 +0100
@@ -55,7 +55,7 @@
BadConnectionId, Unauthorized, ValidationError,
RepositoryError, UniqueTogetherError, onevent)
from cubicweb import cwvreg, schema, server
-from cubicweb.server import ShuttingDown, utils, hook, pool, querier, sources
+from cubicweb.server import ShuttingDown, utils, hook, querier, sources
from cubicweb.server.session import Session, InternalSession, InternalManager
from cubicweb.server.ssplanner import EditedEntity
@@ -219,7 +219,7 @@
self._cnxsets_pool = Queue.Queue()
# 0. init a cnxset that will be used to fetch bootstrap information from
# the database
- self._cnxsets_pool.put_nowait(pool.ConnectionsSet(self.system_source))
+ self._cnxsets_pool.put_nowait(self.system_source.wrapped_connection())
# 1. set used cubes
if config.creating or not config.read_instance_schema:
config.bootstrap_cubes()
@@ -260,7 +260,7 @@
self._get_cnxset().close(True)
self.cnxsets = [] # list of available cnxsets (can't iterate on a Queue)
for i in xrange(config['connections-pool-size']):
- self.cnxsets.append(pool.ConnectionsSet(self.system_source))
+ self.cnxsets.append(self.system_source.wrapped_connection())
self._cnxsets_pool.put_nowait(self.cnxsets[-1])
# internals ###############################################################
--- a/server/session.py Thu Jun 27 09:18:39 2013 +0200
+++ b/server/session.py Wed Jan 22 15:50:03 2014 +0100
@@ -605,11 +605,6 @@
cnxset = self.repo._get_cnxset()
try:
self.cnxset = cnxset
- try:
- cnxset.cnxset_set()
- except:
- self.cnxset = None
- raise
except:
self.repo._free_cnxset(cnxset)
raise
--- a/server/sources/__init__.py Thu Jun 27 09:18:39 2013 +0200
+++ b/server/sources/__init__.py Wed Jan 22 15:50:03 2014 +0100
@@ -232,13 +232,6 @@
"""open and return a connection to the source"""
raise NotImplementedError(self)
- def check_connection(self, cnx):
- """Check connection validity, return None if the connection is still
- valid else a new connection (called when the connections set using the
- given connection is being attached to a session). Do nothing by default.
- """
- pass
-
def close_source_connections(self):
for cnxset in self.repo.cnxsets:
cnxset.cu = None
--- a/server/sources/native.py Thu Jun 27 09:18:39 2013 +0200
+++ b/server/sources/native.py Wed Jan 22 15:50:03 2014 +0100
@@ -59,7 +59,7 @@
from cubicweb.cwconfig import CubicWebNoAppConfiguration
from cubicweb.server import hook
from cubicweb.server.utils import crypt_password, eschema_eid, verify_and_update
-from cubicweb.server.sqlutils import SQL_PREFIX, SQLAdapterMixIn, SqliteCnxLoggingWrapper
+from cubicweb.server.sqlutils import SQL_PREFIX, SQLAdapterMixIn
from cubicweb.server.rqlannotation import set_qdata
from cubicweb.server.hook import CleanupDeletedEidsCacheOp
from cubicweb.server.edition import EditedEntity
@@ -287,17 +287,6 @@
self._eid_creation_cnx = None
# (etype, attr) / storage mapping
self._storages = {}
- # XXX no_sqlite_wrap trick since we've a sqlite locking pb when
- # running unittest_multisources with the wrapping below
- if self.dbdriver == 'sqlite' and \
- not getattr(repo.config, 'no_sqlite_wrap', False):
- from cubicweb.server.pool import ConnectionsSet
- self.dbhelper.dbname = abspath(self.dbhelper.dbname)
- self.get_connection = lambda: SqliteCnxLoggingWrapper(self)
- self.check_connection = lambda cnx: cnx
- def cnxset_freed(self):
- self.cnx.close()
- ConnectionsSet.cnxset_freed = cnxset_freed
if self.dbdriver == 'sqlite':
self._create_eid = None
self.create_eid = self._create_eid_sqlite
@@ -334,7 +323,6 @@
if self.do_fti:
if cnxset is None:
_cnxset = self.repo._get_cnxset()
- _cnxset.cnxset_set()
else:
_cnxset = cnxset
if not self.dbhelper.has_fti_table(_cnxset.cu):
@@ -1599,6 +1587,7 @@
authinfo['email_auth'] = True
return self.source.repo.check_auth_info(session, login, authinfo)
+
class DatabaseIndependentBackupRestore(object):
"""Helper class to perform db backend agnostic backup and restore
--- a/server/sqlutils.py Thu Jun 27 09:18:39 2013 +0200
+++ b/server/sqlutils.py Wed Jan 22 15:50:03 2014 +0100
@@ -1,4 +1,4 @@
-# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# copyright 2003-2014 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
@@ -19,23 +19,25 @@
__docformat__ = "restructuredtext en"
+import sys
import os
import re
import subprocess
-from datetime import datetime, date
+from os.path import abspath
from itertools import ifilter
+from logging import getLogger
from logilab import database as db, common as lgc
from logilab.common.shellutils import ProgressBar
-from logilab.common.date import todate, todatetime, utcdatetime, utctime
+from logilab.common.deprecation import deprecated
+from logilab.common.logging_ext import set_log_methods
from logilab.database.sqlgen import SQLGenerator
-from cubicweb import Binary, ConfigurationError, server
+from cubicweb import Binary, ConfigurationError
from cubicweb.uilib import remove_html_tags
from cubicweb.schema import PURE_VIRTUAL_RTYPES
from cubicweb.server import SQL_CONNECT_HOOKS
from cubicweb.server.utils import crypt_password
-from rql.utils import RQL_FUNCTIONS_REGISTRY
lgc.USE_MX_DATETIME = False
SQL_PREFIX = 'cw_'
@@ -178,10 +180,124 @@
return '\n'.join(cmds)
+class ConnectionWrapper(object):
+ """handle connection to the system source, at some point associated to a
+ :class:`Session`
+ """
+
+ # since 3.19, we only have to manage the system source connection
+ def __init__(self, system_source):
+ # dictionary of (source, connection), indexed by sources'uri
+ self._source = system_source
+ self.cnx = system_source.get_connection()
+ self.cu = self.cnx.cursor()
+
+ def commit(self):
+ """commit the current transaction for this user"""
+ # let exception propagates
+ self.cnx.commit()
+
+ def rollback(self):
+ """rollback the current transaction for this user"""
+ # catch exceptions, rollback other sources anyway
+ try:
+ self.cnx.rollback()
+ except Exception:
+ self._source.critical('rollback error', exc_info=sys.exc_info())
+ # error on rollback, the connection is much probably in a really
+ # bad state. Replace it by a new one.
+ self.reconnect()
+
+ def close(self, i_know_what_i_do=False):
+ """close all connections in the set"""
+ if i_know_what_i_do is not True: # unexpected closing safety belt
+ raise RuntimeError('connections set shouldn\'t be closed')
+ try:
+ self.cu.close()
+ self.cu = None
+ except Exception:
+ pass
+ try:
+ self.cnx.close()
+ self.cnx = None
+ except Exception:
+ pass
+
+ # internals ###############################################################
+
+ def cnxset_freed(self):
+ """connections set is being freed from a session"""
+ pass # no nothing by default
+
+ def reconnect(self):
+ """reopen a connection for this source or all sources if none specified
+ """
+ try:
+ # properly close existing connection if any
+ self.cnx.close()
+ except Exception:
+ pass
+ self._source.info('trying to reconnect')
+ self.cnx = self._source.get_connection()
+ self.cu = self.cnx.cursor()
+
+ @deprecated('[3.19] use .cu instead')
+ def __getitem__(self, uri):
+ assert uri == 'system'
+ return self.cu
+
+ @deprecated('[3.19] use repo.system_source instead')
+ def source(self, uid):
+ assert uid == 'system'
+ return self._source
+
+ @deprecated('[3.19] use .cnx instead')
+ def connection(self, uid):
+ assert uid == 'system'
+ return self.cnx
+
+
+class SqliteConnectionWrapper(ConnectionWrapper):
+ """Sqlite specific connection wrapper: close the connection each time it's
+ freed (and reopen it later when needed)
+ """
+ def __init__(self, system_source):
+ # don't call parent's __init__, we don't want to initiate the connection
+ self._source = system_source
+
+ _cnx = None
+
+ def cnxset_freed(self):
+ self.cu.close()
+ self.cnx.close()
+ self.cnx = self.cu = None
+
+ @property
+ def cnx(self):
+ if self._cnx is None:
+ self._cnx = self._source.get_connection()
+ self._cu = self._cnx.cursor()
+ return self._cnx
+ @cnx.setter
+ def cnx(self, value):
+ self._cnx = value
+
+ @property
+ def cu(self):
+ if self._cnx is None:
+ self._cnx = self._source.get_connection()
+ self._cu = self._cnx.cursor()
+ return self._cu
+ @cu.setter
+ def cu(self, value):
+ self._cu = value
+
+
class SQLAdapterMixIn(object):
"""Mixin for SQL data sources, getting a connection from a configuration
dictionary and handling connection locking
"""
+ cnx_wrap = ConnectionWrapper
def __init__(self, source_config):
try:
@@ -209,6 +325,15 @@
self._binary = self.dbhelper.binary_value
self._process_value = dbapi_module.process_value
self._dbencoding = dbencoding
+ if self.dbdriver == 'sqlite':
+ self.cnx_wrap = SqliteConnectionWrapper
+ self.dbhelper.dbname = abspath(self.dbhelper.dbname)
+
+ def wrapped_connection(self):
+ """open and return a connection to the database, wrapped into a class
+ handling reconnection and all
+ """
+ return self.cnx_wrap(self)
def get_connection(self):
"""open and return a connection to the database"""
@@ -320,46 +445,10 @@
# only defining here to prevent pylint from complaining
info = warning = error = critical = exception = debug = lambda msg,*a,**kw: None
-from logging import getLogger
-from cubicweb import set_log_methods
set_log_methods(SQLAdapterMixIn, getLogger('cubicweb.sqladapter'))
-class SqliteCnxLoggingWrapper(object):
- def __init__(self, source=None):
- self.source = source
- self._cnx = None
-
- def cursor(self):
- # sqlite connections can only be used in the same thread, so
- # create a new one each time necessary. If it appears to be time
- # consuming, find another way
- if self._cnx is None:
- # direct access to SQLAdapterMixIn to get an unwrapped connection
- self._cnx = SQLAdapterMixIn.get_connection(self.source)
- if server.DEBUG & server.DBG_SQL:
- print 'sql cnx OPEN', self._cnx
- return self._cnx.cursor()
-
- def commit(self):
- if self._cnx is not None:
- if server.DEBUG & (server.DBG_SQL | server.DBG_RQL):
- print 'sql cnx COMMIT', self._cnx
- self._cnx.commit()
-
- def rollback(self):
- if self._cnx is not None:
- if server.DEBUG & (server.DBG_SQL | server.DBG_RQL):
- print 'sql cnx ROLLBACK', self._cnx
- self._cnx.rollback()
-
- def close(self):
- if self._cnx is not None:
- if server.DEBUG & server.DBG_SQL:
- print 'sql cnx CLOSE', self._cnx
- self._cnx.close()
- self._cnx = None
-
+# connection initialization functions ##########################################
def init_sqlite_connexion(cnx):