[multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper
authorSylvain Thénault <sylvain.thenault@logilab.fr>
Wed, 22 Jan 2014 15:50:03 +0100
changeset 9466 c3a5f4507f12
parent 9465 86d4b41ae339
child 9467 ad66d7b3fd48
[multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper also, SqliteConnectionWrapper becomes a subclass. This is allowed since now ConnectionsSet responsability has been reduced to handling the system source only. This changeset also: * drops useless cnxset_set / check_connection api * deprecates former .source / .connection / container api but it renames neither the session's cnxset attribute, nor the related repository method. Related to #2919300
server/pool.py
server/repository.py
server/session.py
server/sources/__init__.py
server/sources/native.py
server/sqlutils.py
--- a/server/pool.py	Thu Jun 27 09:18:39 2013 +0200
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,126 +0,0 @@
-# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
-# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
-#
-# This file is part of CubicWeb.
-#
-# CubicWeb is free software: you can redistribute it and/or modify it under the
-# terms of the GNU Lesser General Public License as published by the Free
-# Software Foundation, either version 2.1 of the License, or (at your option)
-# any later version.
-#
-# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
-# details.
-#
-# You should have received a copy of the GNU Lesser General Public License along
-# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
-"""CubicWeb server connections set : the repository has a limited number of
-:class:`ConnectionsSet` (defined in configuration, default to 4). Each of them
-hold a connection to the system source.
-"""
-
-__docformat__ = "restructuredtext en"
-
-import sys
-
-from logilab.common.deprecation import deprecated
-
-
-class ConnectionsSet(object):
-    """handle connection to the system source, at some point associated to a
-    :class:`Session`
-    """
-
-    # since 3.19, we only have to manage the system source connection
-    def __init__(self, system_source):
-        # dictionary of (source, connection), indexed by sources'uri
-        self._source = system_source
-        self.cnx = system_source.get_connection()
-        self.cu = self.cnx.cursor()
-
-    def commit(self):
-        """commit the current transaction for this user"""
-        # let exception propagates
-        self.cnx.commit()
-
-    def rollback(self):
-        """rollback the current transaction for this user"""
-        # catch exceptions, rollback other sources anyway
-        try:
-            self.cnx.rollback()
-        except Exception:
-            self._source.critical('rollback error', exc_info=sys.exc_info())
-            # error on rollback, the connection is much probably in a really
-            # bad state. Replace it by a new one.
-            self.reconnect()
-
-    def close(self, i_know_what_i_do=False):
-        """close all connections in the set"""
-        if i_know_what_i_do is not True: # unexpected closing safety belt
-            raise RuntimeError('connections set shouldn\'t be closed')
-        try:
-            self.cu.close()
-            self.cu = None
-        except Exception:
-            pass
-        try:
-            self.cnx.close()
-        except Exception:
-            pass
-
-    # internals ###############################################################
-
-    def cnxset_set(self):
-        """connections set is being set on a session"""
-        self.check_connections()
-
-    def cnxset_freed(self):
-        """connections set is being freed from a session"""
-        pass # do nothing by default
-
-    def reconnect(self):
-        """reopen a connection for this source or all sources if none specified
-        """
-        try:
-            # properly close existing connection if any
-            self.cnx.close()
-        except Exception:
-            pass
-        self._source.info('trying to reconnect')
-        self.cnx = self._source.get_connection()
-        self.cu = self.cnx.cursor()
-
-    def check_connections(self):
-        newcnx = self._source.check_connection(self.cnx)
-        if newcnx is not None:
-            self.cnx = newcnx
-            self.cu = self.cnx.cursor()
-
-    @deprecated('[3.19] use .cu instead')
-    def __getitem__(self, uri):
-        assert uri == 'system'
-        return self.cu
-
-    @deprecated('[3.19] use repo.system_source instead')
-    def source(self, uid):
-        assert uid == 'system'
-        return self._source
-
-    @deprecated('[3.19] use .cnx instead')
-    def connection(self, uid):
-        assert uid == 'system'
-        return self.cnx
-
-    @property
-    @deprecated('[3.19] use .cnx instead')
-    def source_cnxs(self):
-        return {'system': (self._source, self.cnx)}
-
-
-from cubicweb.server.hook import Operation, LateOperation, SingleLastOperation
-from logilab.common.deprecation import class_moved, class_renamed
-Operation = class_moved(Operation)
-PreCommitOperation = class_renamed('PreCommitOperation', Operation)
-LateOperation = class_moved(LateOperation)
-SingleLastOperation = class_moved(SingleLastOperation)
--- a/server/repository.py	Thu Jun 27 09:18:39 2013 +0200
+++ b/server/repository.py	Wed Jan 22 15:50:03 2014 +0100
@@ -55,7 +55,7 @@
                       BadConnectionId, Unauthorized, ValidationError,
                       RepositoryError, UniqueTogetherError, onevent)
 from cubicweb import cwvreg, schema, server
-from cubicweb.server import ShuttingDown, utils, hook, pool, querier, sources
+from cubicweb.server import ShuttingDown, utils, hook, querier, sources
 from cubicweb.server.session import Session, InternalSession, InternalManager
 from cubicweb.server.ssplanner import EditedEntity
 
@@ -219,7 +219,7 @@
         self._cnxsets_pool = Queue.Queue()
         # 0. init a cnxset that will be used to fetch bootstrap information from
         #    the database
-        self._cnxsets_pool.put_nowait(pool.ConnectionsSet(self.system_source))
+        self._cnxsets_pool.put_nowait(self.system_source.wrapped_connection())
         # 1. set used cubes
         if config.creating or not config.read_instance_schema:
             config.bootstrap_cubes()
@@ -260,7 +260,7 @@
         self._get_cnxset().close(True)
         self.cnxsets = [] # list of available cnxsets (can't iterate on a Queue)
         for i in xrange(config['connections-pool-size']):
-            self.cnxsets.append(pool.ConnectionsSet(self.system_source))
+            self.cnxsets.append(self.system_source.wrapped_connection())
             self._cnxsets_pool.put_nowait(self.cnxsets[-1])
 
     # internals ###############################################################
--- a/server/session.py	Thu Jun 27 09:18:39 2013 +0200
+++ b/server/session.py	Wed Jan 22 15:50:03 2014 +0100
@@ -605,11 +605,6 @@
             cnxset = self.repo._get_cnxset()
             try:
                 self.cnxset = cnxset
-                try:
-                    cnxset.cnxset_set()
-                except:
-                    self.cnxset = None
-                    raise
             except:
                 self.repo._free_cnxset(cnxset)
                 raise
--- a/server/sources/__init__.py	Thu Jun 27 09:18:39 2013 +0200
+++ b/server/sources/__init__.py	Wed Jan 22 15:50:03 2014 +0100
@@ -232,13 +232,6 @@
         """open and return a connection to the source"""
         raise NotImplementedError(self)
 
-    def check_connection(self, cnx):
-        """Check connection validity, return None if the connection is still
-        valid else a new connection (called when the connections set using the
-        given connection is being attached to a session). Do nothing by default.
-        """
-        pass
-
     def close_source_connections(self):
         for cnxset in self.repo.cnxsets:
             cnxset.cu = None
--- a/server/sources/native.py	Thu Jun 27 09:18:39 2013 +0200
+++ b/server/sources/native.py	Wed Jan 22 15:50:03 2014 +0100
@@ -59,7 +59,7 @@
 from cubicweb.cwconfig import CubicWebNoAppConfiguration
 from cubicweb.server import hook
 from cubicweb.server.utils import crypt_password, eschema_eid, verify_and_update
-from cubicweb.server.sqlutils import SQL_PREFIX, SQLAdapterMixIn, SqliteCnxLoggingWrapper
+from cubicweb.server.sqlutils import SQL_PREFIX, SQLAdapterMixIn
 from cubicweb.server.rqlannotation import set_qdata
 from cubicweb.server.hook import CleanupDeletedEidsCacheOp
 from cubicweb.server.edition import EditedEntity
@@ -287,17 +287,6 @@
         self._eid_creation_cnx = None
         # (etype, attr) / storage mapping
         self._storages = {}
-        # XXX no_sqlite_wrap trick since we've a sqlite locking pb when
-        # running unittest_multisources with the wrapping below
-        if self.dbdriver == 'sqlite' and \
-               not getattr(repo.config, 'no_sqlite_wrap', False):
-            from cubicweb.server.pool import ConnectionsSet
-            self.dbhelper.dbname = abspath(self.dbhelper.dbname)
-            self.get_connection = lambda: SqliteCnxLoggingWrapper(self)
-            self.check_connection = lambda cnx: cnx
-            def cnxset_freed(self):
-                self.cnx.close()
-            ConnectionsSet.cnxset_freed = cnxset_freed
         if self.dbdriver == 'sqlite':
             self._create_eid = None
             self.create_eid = self._create_eid_sqlite
@@ -334,7 +323,6 @@
         if self.do_fti:
             if cnxset is None:
                 _cnxset = self.repo._get_cnxset()
-                _cnxset.cnxset_set()
             else:
                 _cnxset = cnxset
             if not self.dbhelper.has_fti_table(_cnxset.cu):
@@ -1599,6 +1587,7 @@
         authinfo['email_auth'] = True
         return self.source.repo.check_auth_info(session, login, authinfo)
 
+
 class DatabaseIndependentBackupRestore(object):
     """Helper class to perform db backend agnostic backup and restore
 
--- a/server/sqlutils.py	Thu Jun 27 09:18:39 2013 +0200
+++ b/server/sqlutils.py	Wed Jan 22 15:50:03 2014 +0100
@@ -1,4 +1,4 @@
-# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# copyright 2003-2014 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
 # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
 #
 # This file is part of CubicWeb.
@@ -19,23 +19,25 @@
 
 __docformat__ = "restructuredtext en"
 
+import sys
 import os
 import re
 import subprocess
-from datetime import datetime, date
+from os.path import abspath
 from itertools import ifilter
+from logging import getLogger
 
 from logilab import database as db, common as lgc
 from logilab.common.shellutils import ProgressBar
-from logilab.common.date import todate, todatetime, utcdatetime, utctime
+from logilab.common.deprecation import deprecated
+from logilab.common.logging_ext import set_log_methods
 from logilab.database.sqlgen import SQLGenerator
 
-from cubicweb import Binary, ConfigurationError, server
+from cubicweb import Binary, ConfigurationError
 from cubicweb.uilib import remove_html_tags
 from cubicweb.schema import PURE_VIRTUAL_RTYPES
 from cubicweb.server import SQL_CONNECT_HOOKS
 from cubicweb.server.utils import crypt_password
-from rql.utils import RQL_FUNCTIONS_REGISTRY
 
 lgc.USE_MX_DATETIME = False
 SQL_PREFIX = 'cw_'
@@ -178,10 +180,124 @@
     return '\n'.join(cmds)
 
 
+class ConnectionWrapper(object):
+    """handle connection to the system source, at some point associated to a
+    :class:`Session`
+    """
+
+    # since 3.19, we only have to manage the system source connection
+    def __init__(self, system_source):
+        # dictionary of (source, connection), indexed by sources'uri
+        self._source = system_source
+        self.cnx = system_source.get_connection()
+        self.cu = self.cnx.cursor()
+
+    def commit(self):
+        """commit the current transaction for this user"""
+        # let exception propagates
+        self.cnx.commit()
+
+    def rollback(self):
+        """rollback the current transaction for this user"""
+        # catch exceptions, rollback other sources anyway
+        try:
+            self.cnx.rollback()
+        except Exception:
+            self._source.critical('rollback error', exc_info=sys.exc_info())
+            # error on rollback, the connection is much probably in a really
+            # bad state. Replace it by a new one.
+            self.reconnect()
+
+    def close(self, i_know_what_i_do=False):
+        """close all connections in the set"""
+        if i_know_what_i_do is not True: # unexpected closing safety belt
+            raise RuntimeError('connections set shouldn\'t be closed')
+        try:
+            self.cu.close()
+            self.cu = None
+        except Exception:
+            pass
+        try:
+            self.cnx.close()
+            self.cnx = None
+        except Exception:
+            pass
+
+    # internals ###############################################################
+
+    def cnxset_freed(self):
+        """connections set is being freed from a session"""
+        pass # no nothing by default
+
+    def reconnect(self):
+        """reopen a connection for this source or all sources if none specified
+        """
+        try:
+            # properly close existing connection if any
+            self.cnx.close()
+        except Exception:
+            pass
+        self._source.info('trying to reconnect')
+        self.cnx = self._source.get_connection()
+        self.cu = self.cnx.cursor()
+
+    @deprecated('[3.19] use .cu instead')
+    def __getitem__(self, uri):
+        assert uri == 'system'
+        return self.cu
+
+    @deprecated('[3.19] use repo.system_source instead')
+    def source(self, uid):
+        assert uid == 'system'
+        return self._source
+
+    @deprecated('[3.19] use .cnx instead')
+    def connection(self, uid):
+        assert uid == 'system'
+        return self.cnx
+
+
+class SqliteConnectionWrapper(ConnectionWrapper):
+    """Sqlite specific connection wrapper: close the connection each time it's
+    freed (and reopen it later when needed)
+    """
+    def __init__(self, system_source):
+        # don't call parent's __init__, we don't want to initiate the connection
+        self._source = system_source
+
+    _cnx = None
+
+    def cnxset_freed(self):
+        self.cu.close()
+        self.cnx.close()
+        self.cnx = self.cu = None
+
+    @property
+    def cnx(self):
+        if self._cnx is None:
+            self._cnx = self._source.get_connection()
+            self._cu = self._cnx.cursor()
+        return self._cnx
+    @cnx.setter
+    def cnx(self, value):
+        self._cnx = value
+
+    @property
+    def cu(self):
+        if self._cnx is None:
+            self._cnx = self._source.get_connection()
+            self._cu = self._cnx.cursor()
+        return self._cu
+    @cu.setter
+    def cu(self, value):
+        self._cu = value
+
+
 class SQLAdapterMixIn(object):
     """Mixin for SQL data sources, getting a connection from a configuration
     dictionary and handling connection locking
     """
+    cnx_wrap = ConnectionWrapper
 
     def __init__(self, source_config):
         try:
@@ -209,6 +325,15 @@
         self._binary = self.dbhelper.binary_value
         self._process_value = dbapi_module.process_value
         self._dbencoding = dbencoding
+        if self.dbdriver == 'sqlite':
+            self.cnx_wrap = SqliteConnectionWrapper
+            self.dbhelper.dbname = abspath(self.dbhelper.dbname)
+
+    def wrapped_connection(self):
+        """open and return a connection to the database, wrapped into a class
+        handling reconnection and all
+        """
+        return self.cnx_wrap(self)
 
     def get_connection(self):
         """open and return a connection to the database"""
@@ -320,46 +445,10 @@
     # only defining here to prevent pylint from complaining
     info = warning = error = critical = exception = debug = lambda msg,*a,**kw: None
 
-from logging import getLogger
-from cubicweb import set_log_methods
 set_log_methods(SQLAdapterMixIn, getLogger('cubicweb.sqladapter'))
 
 
-class SqliteCnxLoggingWrapper(object):
-    def __init__(self, source=None):
-        self.source = source
-        self._cnx = None
-
-    def cursor(self):
-        # sqlite connections can only be used in the same thread, so
-        # create a new one each time necessary. If it appears to be time
-        # consuming, find another way
-        if self._cnx is None:
-            # direct access to SQLAdapterMixIn to get an unwrapped connection
-            self._cnx = SQLAdapterMixIn.get_connection(self.source)
-            if server.DEBUG & server.DBG_SQL:
-                print 'sql cnx OPEN', self._cnx
-        return self._cnx.cursor()
-
-    def commit(self):
-        if self._cnx is not None:
-            if server.DEBUG & (server.DBG_SQL | server.DBG_RQL):
-                print 'sql cnx COMMIT', self._cnx
-            self._cnx.commit()
-
-    def rollback(self):
-        if self._cnx is not None:
-            if server.DEBUG & (server.DBG_SQL | server.DBG_RQL):
-                print 'sql cnx ROLLBACK', self._cnx
-            self._cnx.rollback()
-
-    def close(self):
-        if self._cnx is not None:
-            if server.DEBUG & server.DBG_SQL:
-                print 'sql cnx CLOSE', self._cnx
-            self._cnx.close()
-            self._cnx = None
-
+# connection initialization functions ##########################################
 
 def init_sqlite_connexion(cnx):