# HG changeset patch # User Sylvain Thénault # Date 1390402203 -3600 # Node ID c3a5f4507f12537333411a82ba65281742591a5a # Parent 86d4b41ae339d5cbad30d419ba22b058778479fc [multi-sources-removal] Turn ConnectionsSet into simpler ConnectionWrapper also, SqliteConnectionWrapper becomes a subclass. This is allowed since now ConnectionsSet responsability has been reduced to handling the system source only. This changeset also: * drops useless cnxset_set / check_connection api * deprecates former .source / .connection / container api but it renames neither the session's cnxset attribute, nor the related repository method. Related to #2919300 diff -r 86d4b41ae339 -r c3a5f4507f12 server/pool.py --- a/server/pool.py Thu Jun 27 09:18:39 2013 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,126 +0,0 @@ -# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved. -# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr -# -# This file is part of CubicWeb. -# -# CubicWeb is free software: you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation, either version 2.1 of the License, or (at your option) -# any later version. -# -# CubicWeb is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License along -# with CubicWeb. If not, see . -"""CubicWeb server connections set : the repository has a limited number of -:class:`ConnectionsSet` (defined in configuration, default to 4). Each of them -hold a connection to the system source. -""" - -__docformat__ = "restructuredtext en" - -import sys - -from logilab.common.deprecation import deprecated - - -class ConnectionsSet(object): - """handle connection to the system source, at some point associated to a - :class:`Session` - """ - - # since 3.19, we only have to manage the system source connection - def __init__(self, system_source): - # dictionary of (source, connection), indexed by sources'uri - self._source = system_source - self.cnx = system_source.get_connection() - self.cu = self.cnx.cursor() - - def commit(self): - """commit the current transaction for this user""" - # let exception propagates - self.cnx.commit() - - def rollback(self): - """rollback the current transaction for this user""" - # catch exceptions, rollback other sources anyway - try: - self.cnx.rollback() - except Exception: - self._source.critical('rollback error', exc_info=sys.exc_info()) - # error on rollback, the connection is much probably in a really - # bad state. Replace it by a new one. - self.reconnect() - - def close(self, i_know_what_i_do=False): - """close all connections in the set""" - if i_know_what_i_do is not True: # unexpected closing safety belt - raise RuntimeError('connections set shouldn\'t be closed') - try: - self.cu.close() - self.cu = None - except Exception: - pass - try: - self.cnx.close() - except Exception: - pass - - # internals ############################################################### - - def cnxset_set(self): - """connections set is being set on a session""" - self.check_connections() - - def cnxset_freed(self): - """connections set is being freed from a session""" - pass # do nothing by default - - def reconnect(self): - """reopen a connection for this source or all sources if none specified - """ - try: - # properly close existing connection if any - self.cnx.close() - except Exception: - pass - self._source.info('trying to reconnect') - self.cnx = self._source.get_connection() - self.cu = self.cnx.cursor() - - def check_connections(self): - newcnx = self._source.check_connection(self.cnx) - if newcnx is not None: - self.cnx = newcnx - self.cu = self.cnx.cursor() - - @deprecated('[3.19] use .cu instead') - def __getitem__(self, uri): - assert uri == 'system' - return self.cu - - @deprecated('[3.19] use repo.system_source instead') - def source(self, uid): - assert uid == 'system' - return self._source - - @deprecated('[3.19] use .cnx instead') - def connection(self, uid): - assert uid == 'system' - return self.cnx - - @property - @deprecated('[3.19] use .cnx instead') - def source_cnxs(self): - return {'system': (self._source, self.cnx)} - - -from cubicweb.server.hook import Operation, LateOperation, SingleLastOperation -from logilab.common.deprecation import class_moved, class_renamed -Operation = class_moved(Operation) -PreCommitOperation = class_renamed('PreCommitOperation', Operation) -LateOperation = class_moved(LateOperation) -SingleLastOperation = class_moved(SingleLastOperation) diff -r 86d4b41ae339 -r c3a5f4507f12 server/repository.py --- a/server/repository.py Thu Jun 27 09:18:39 2013 +0200 +++ b/server/repository.py Wed Jan 22 15:50:03 2014 +0100 @@ -55,7 +55,7 @@ BadConnectionId, Unauthorized, ValidationError, RepositoryError, UniqueTogetherError, onevent) from cubicweb import cwvreg, schema, server -from cubicweb.server import ShuttingDown, utils, hook, pool, querier, sources +from cubicweb.server import ShuttingDown, utils, hook, querier, sources from cubicweb.server.session import Session, InternalSession, InternalManager from cubicweb.server.ssplanner import EditedEntity @@ -219,7 +219,7 @@ self._cnxsets_pool = Queue.Queue() # 0. init a cnxset that will be used to fetch bootstrap information from # the database - self._cnxsets_pool.put_nowait(pool.ConnectionsSet(self.system_source)) + self._cnxsets_pool.put_nowait(self.system_source.wrapped_connection()) # 1. set used cubes if config.creating or not config.read_instance_schema: config.bootstrap_cubes() @@ -260,7 +260,7 @@ self._get_cnxset().close(True) self.cnxsets = [] # list of available cnxsets (can't iterate on a Queue) for i in xrange(config['connections-pool-size']): - self.cnxsets.append(pool.ConnectionsSet(self.system_source)) + self.cnxsets.append(self.system_source.wrapped_connection()) self._cnxsets_pool.put_nowait(self.cnxsets[-1]) # internals ############################################################### diff -r 86d4b41ae339 -r c3a5f4507f12 server/session.py --- a/server/session.py Thu Jun 27 09:18:39 2013 +0200 +++ b/server/session.py Wed Jan 22 15:50:03 2014 +0100 @@ -605,11 +605,6 @@ cnxset = self.repo._get_cnxset() try: self.cnxset = cnxset - try: - cnxset.cnxset_set() - except: - self.cnxset = None - raise except: self.repo._free_cnxset(cnxset) raise diff -r 86d4b41ae339 -r c3a5f4507f12 server/sources/__init__.py --- a/server/sources/__init__.py Thu Jun 27 09:18:39 2013 +0200 +++ b/server/sources/__init__.py Wed Jan 22 15:50:03 2014 +0100 @@ -232,13 +232,6 @@ """open and return a connection to the source""" raise NotImplementedError(self) - def check_connection(self, cnx): - """Check connection validity, return None if the connection is still - valid else a new connection (called when the connections set using the - given connection is being attached to a session). Do nothing by default. - """ - pass - def close_source_connections(self): for cnxset in self.repo.cnxsets: cnxset.cu = None diff -r 86d4b41ae339 -r c3a5f4507f12 server/sources/native.py --- a/server/sources/native.py Thu Jun 27 09:18:39 2013 +0200 +++ b/server/sources/native.py Wed Jan 22 15:50:03 2014 +0100 @@ -59,7 +59,7 @@ from cubicweb.cwconfig import CubicWebNoAppConfiguration from cubicweb.server import hook from cubicweb.server.utils import crypt_password, eschema_eid, verify_and_update -from cubicweb.server.sqlutils import SQL_PREFIX, SQLAdapterMixIn, SqliteCnxLoggingWrapper +from cubicweb.server.sqlutils import SQL_PREFIX, SQLAdapterMixIn from cubicweb.server.rqlannotation import set_qdata from cubicweb.server.hook import CleanupDeletedEidsCacheOp from cubicweb.server.edition import EditedEntity @@ -287,17 +287,6 @@ self._eid_creation_cnx = None # (etype, attr) / storage mapping self._storages = {} - # XXX no_sqlite_wrap trick since we've a sqlite locking pb when - # running unittest_multisources with the wrapping below - if self.dbdriver == 'sqlite' and \ - not getattr(repo.config, 'no_sqlite_wrap', False): - from cubicweb.server.pool import ConnectionsSet - self.dbhelper.dbname = abspath(self.dbhelper.dbname) - self.get_connection = lambda: SqliteCnxLoggingWrapper(self) - self.check_connection = lambda cnx: cnx - def cnxset_freed(self): - self.cnx.close() - ConnectionsSet.cnxset_freed = cnxset_freed if self.dbdriver == 'sqlite': self._create_eid = None self.create_eid = self._create_eid_sqlite @@ -334,7 +323,6 @@ if self.do_fti: if cnxset is None: _cnxset = self.repo._get_cnxset() - _cnxset.cnxset_set() else: _cnxset = cnxset if not self.dbhelper.has_fti_table(_cnxset.cu): @@ -1599,6 +1587,7 @@ authinfo['email_auth'] = True return self.source.repo.check_auth_info(session, login, authinfo) + class DatabaseIndependentBackupRestore(object): """Helper class to perform db backend agnostic backup and restore diff -r 86d4b41ae339 -r c3a5f4507f12 server/sqlutils.py --- a/server/sqlutils.py Thu Jun 27 09:18:39 2013 +0200 +++ b/server/sqlutils.py Wed Jan 22 15:50:03 2014 +0100 @@ -1,4 +1,4 @@ -# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved. +# copyright 2003-2014 LOGILAB S.A. (Paris, FRANCE), all rights reserved. # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr # # This file is part of CubicWeb. @@ -19,23 +19,25 @@ __docformat__ = "restructuredtext en" +import sys import os import re import subprocess -from datetime import datetime, date +from os.path import abspath from itertools import ifilter +from logging import getLogger from logilab import database as db, common as lgc from logilab.common.shellutils import ProgressBar -from logilab.common.date import todate, todatetime, utcdatetime, utctime +from logilab.common.deprecation import deprecated +from logilab.common.logging_ext import set_log_methods from logilab.database.sqlgen import SQLGenerator -from cubicweb import Binary, ConfigurationError, server +from cubicweb import Binary, ConfigurationError from cubicweb.uilib import remove_html_tags from cubicweb.schema import PURE_VIRTUAL_RTYPES from cubicweb.server import SQL_CONNECT_HOOKS from cubicweb.server.utils import crypt_password -from rql.utils import RQL_FUNCTIONS_REGISTRY lgc.USE_MX_DATETIME = False SQL_PREFIX = 'cw_' @@ -178,10 +180,124 @@ return '\n'.join(cmds) +class ConnectionWrapper(object): + """handle connection to the system source, at some point associated to a + :class:`Session` + """ + + # since 3.19, we only have to manage the system source connection + def __init__(self, system_source): + # dictionary of (source, connection), indexed by sources'uri + self._source = system_source + self.cnx = system_source.get_connection() + self.cu = self.cnx.cursor() + + def commit(self): + """commit the current transaction for this user""" + # let exception propagates + self.cnx.commit() + + def rollback(self): + """rollback the current transaction for this user""" + # catch exceptions, rollback other sources anyway + try: + self.cnx.rollback() + except Exception: + self._source.critical('rollback error', exc_info=sys.exc_info()) + # error on rollback, the connection is much probably in a really + # bad state. Replace it by a new one. + self.reconnect() + + def close(self, i_know_what_i_do=False): + """close all connections in the set""" + if i_know_what_i_do is not True: # unexpected closing safety belt + raise RuntimeError('connections set shouldn\'t be closed') + try: + self.cu.close() + self.cu = None + except Exception: + pass + try: + self.cnx.close() + self.cnx = None + except Exception: + pass + + # internals ############################################################### + + def cnxset_freed(self): + """connections set is being freed from a session""" + pass # no nothing by default + + def reconnect(self): + """reopen a connection for this source or all sources if none specified + """ + try: + # properly close existing connection if any + self.cnx.close() + except Exception: + pass + self._source.info('trying to reconnect') + self.cnx = self._source.get_connection() + self.cu = self.cnx.cursor() + + @deprecated('[3.19] use .cu instead') + def __getitem__(self, uri): + assert uri == 'system' + return self.cu + + @deprecated('[3.19] use repo.system_source instead') + def source(self, uid): + assert uid == 'system' + return self._source + + @deprecated('[3.19] use .cnx instead') + def connection(self, uid): + assert uid == 'system' + return self.cnx + + +class SqliteConnectionWrapper(ConnectionWrapper): + """Sqlite specific connection wrapper: close the connection each time it's + freed (and reopen it later when needed) + """ + def __init__(self, system_source): + # don't call parent's __init__, we don't want to initiate the connection + self._source = system_source + + _cnx = None + + def cnxset_freed(self): + self.cu.close() + self.cnx.close() + self.cnx = self.cu = None + + @property + def cnx(self): + if self._cnx is None: + self._cnx = self._source.get_connection() + self._cu = self._cnx.cursor() + return self._cnx + @cnx.setter + def cnx(self, value): + self._cnx = value + + @property + def cu(self): + if self._cnx is None: + self._cnx = self._source.get_connection() + self._cu = self._cnx.cursor() + return self._cu + @cu.setter + def cu(self, value): + self._cu = value + + class SQLAdapterMixIn(object): """Mixin for SQL data sources, getting a connection from a configuration dictionary and handling connection locking """ + cnx_wrap = ConnectionWrapper def __init__(self, source_config): try: @@ -209,6 +325,15 @@ self._binary = self.dbhelper.binary_value self._process_value = dbapi_module.process_value self._dbencoding = dbencoding + if self.dbdriver == 'sqlite': + self.cnx_wrap = SqliteConnectionWrapper + self.dbhelper.dbname = abspath(self.dbhelper.dbname) + + def wrapped_connection(self): + """open and return a connection to the database, wrapped into a class + handling reconnection and all + """ + return self.cnx_wrap(self) def get_connection(self): """open and return a connection to the database""" @@ -320,46 +445,10 @@ # only defining here to prevent pylint from complaining info = warning = error = critical = exception = debug = lambda msg,*a,**kw: None -from logging import getLogger -from cubicweb import set_log_methods set_log_methods(SQLAdapterMixIn, getLogger('cubicweb.sqladapter')) -class SqliteCnxLoggingWrapper(object): - def __init__(self, source=None): - self.source = source - self._cnx = None - - def cursor(self): - # sqlite connections can only be used in the same thread, so - # create a new one each time necessary. If it appears to be time - # consuming, find another way - if self._cnx is None: - # direct access to SQLAdapterMixIn to get an unwrapped connection - self._cnx = SQLAdapterMixIn.get_connection(self.source) - if server.DEBUG & server.DBG_SQL: - print 'sql cnx OPEN', self._cnx - return self._cnx.cursor() - - def commit(self): - if self._cnx is not None: - if server.DEBUG & (server.DBG_SQL | server.DBG_RQL): - print 'sql cnx COMMIT', self._cnx - self._cnx.commit() - - def rollback(self): - if self._cnx is not None: - if server.DEBUG & (server.DBG_SQL | server.DBG_RQL): - print 'sql cnx ROLLBACK', self._cnx - self._cnx.rollback() - - def close(self): - if self._cnx is not None: - if server.DEBUG & server.DBG_SQL: - print 'sql cnx CLOSE', self._cnx - self._cnx.close() - self._cnx = None - +# connection initialization functions ########################################## def init_sqlite_connexion(cnx):