1 # copyright 2003-2015 LOGILAB S.A. (Paris, FRANCE), all rights reserved.

     2 # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr

     3 #

     4 # This file is part of CubicWeb.

     5 #

     6 # CubicWeb is free software: you can redistribute it and/or modify it under the

     7 # terms of the GNU Lesser General Public License as published by the Free

     8 # Software Foundation, either version 2.1 of the License, or (at your option)

     9 # any later version.

    10 #

    11 # CubicWeb is distributed in the hope that it will be useful, but WITHOUT

    12 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS

    13 # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more

    14 # details.

    15 #

    16 # You should have received a copy of the GNU Lesser General Public License along

    17 # with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.

    18 """Test tools for cubicweb"""

    19 from __future__ import print_function

    20 

    21 __docformat__ = "restructuredtext en"

    22 

    23 import os

    24 import sys

    25 import errno

    26 import logging

    27 import shutil

    28 import glob

    29 import subprocess

    30 import warnings

    31 import tempfile

    32 import getpass

    33 from hashlib import sha1  # pylint: disable=E0611

    34 from datetime import timedelta

    35 from os.path import abspath, join, exists, split, isabs, isdir

    36 from functools import partial

    37 

    38 from six import text_type

    39 from six.moves import cPickle as pickle

    40 

    41 from logilab.common.date import strptime

    42 from logilab.common.decorators import cached, clear_cache

    43 

    44 from cubicweb import ExecutionError, BadConnectionId

    45 from cubicweb import schema, cwconfig

    46 from cubicweb.server.serverconfig import ServerConfiguration

    47 from cubicweb.etwist.twconfig import WebConfigurationBase

    48 

    49 cwconfig.CubicWebConfiguration.cls_adjust_sys_path()

    50 

    51 # db auto-population configuration #############################################

    52 

    53 SYSTEM_ENTITIES = (schema.SCHEMA_TYPES

    54                    | schema.INTERNAL_TYPES

    55                    | schema.WORKFLOW_TYPES

    56                    | set(('CWGroup', 'CWUser',))

    57                    )

    58 SYSTEM_RELATIONS = (schema.META_RTYPES

    59                     | schema.WORKFLOW_RTYPES

    60                     | schema.WORKFLOW_DEF_RTYPES

    61                     | schema.SYSTEM_RTYPES

    62                     | schema.SCHEMA_TYPES

    63                     | set(('primary_email', # deducted from other relations

    64                            ))

    65                     )

    66 

    67 # content validation configuration #############################################

    68 

    69 # validators are used to validate (XML, DTD, whatever) view's content

    70 # validators availables are :

    71 #  'dtd' : validates XML + declared DTD

    72 #  'xml' : guarantees XML is well formed

    73 #  None : do not try to validate anything

    74 

    75 # {'vid': validator}

    76 VIEW_VALIDATORS = {}

    77 

    78 

    79 # cubicweb test configuration ##################################################

    80 

    81 BASE_URL = 'http://testing.fr/cubicweb/'

    82 

    83 DEFAULT_SOURCES = {'system': {'adapter' : 'native',

    84                               'db-encoding' : 'UTF-8', #'ISO-8859-1',

    85                               'db-user' : u'admin',

    86                               'db-password' : 'gingkow',

    87                               'db-name' : 'tmpdb',

    88                               'db-driver' : 'sqlite',

    89                               'db-host' : None,

    90                               },

    91                    'admin' : {'login': u'admin',

    92                               'password': u'gingkow',

    93                               },

    94                    }

    95 DEFAULT_PSQL_SOURCES = DEFAULT_SOURCES.copy()

    96 DEFAULT_PSQL_SOURCES['system'] = DEFAULT_SOURCES['system'].copy()

    97 DEFAULT_PSQL_SOURCES['system']['db-driver'] = 'postgres'

    98 DEFAULT_PSQL_SOURCES['system']['db-user'] = text_type(getpass.getuser())

    99 DEFAULT_PSQL_SOURCES['system']['db-password'] = None

   100 

   101 def turn_repo_off(repo):

   102     """ Idea: this is less costly than a full re-creation of the repo object.

   103     off:

   104     * session are closed,

   105     * cnxsets are closed

   106     * system source is shutdown

   107     """

   108     if not repo._needs_refresh:

   109         for sessionid in list(repo._sessions):

   110             warnings.warn('%s Open session found while turning repository off'

   111                           %sessionid, RuntimeWarning)

   112             try:

   113                 repo.close(sessionid)

   114             except BadConnectionId: #this is strange ? thread issue ?

   115                 print('XXX unknown session', sessionid)

   116         for cnxset in repo.cnxsets:

   117             cnxset.close(True)

   118         repo.system_source.shutdown()

   119         repo._needs_refresh = True

   120         repo._has_started = False

   121 

   122 

   123 def turn_repo_on(repo):

   124     """Idea: this is less costly than a full re-creation of the repo object.

   125     on:

   126     * cnxsets are connected

   127     * cache are cleared

   128     """

   129     if repo._needs_refresh:

   130         for cnxset in repo.cnxsets:

   131             cnxset.reconnect()

   132         repo._type_source_cache = {}

   133         repo._extid_cache = {}

   134         repo.querier._rql_cache = {}

   135         repo.system_source.reset_caches()

   136         repo._needs_refresh = False

   137 

   138 

   139 class TestServerConfiguration(ServerConfiguration):

   140     mode = 'test'

   141     read_instance_schema = False

   142     init_repository = True

   143     skip_db_create_and_restore = False

   144     default_sources = DEFAULT_SOURCES

   145 

   146     def __init__(self, appid='data', apphome=None, log_threshold=logging.CRITICAL+10):

   147         # must be set before calling parent __init__

   148         if apphome is None:

   149             if exists(appid):

   150                 apphome = abspath(appid)

   151             else: # cube test

   152                 apphome = abspath('..')

   153         self._apphome = apphome

   154         super(TestServerConfiguration, self).__init__(appid)

   155         self.init_log(log_threshold, force=True)

   156         # need this, usually triggered by cubicweb-ctl

   157         self.load_cwctl_plugins()

   158 

   159     # By default anonymous login are allow but some test need to deny of to

   160     # change the default user. Set it to None to prevent anonymous login.

   161     anonymous_credential = ('anon', 'anon')

   162 

   163     def anonymous_user(self):

   164         if not self.anonymous_credential:

   165             return None, None

   166         return self.anonymous_credential

   167 

   168     def set_anonymous_allowed(self, allowed, anonuser=u'anon'):

   169         if allowed:

   170             self.anonymous_credential = (anonuser, anonuser)

   171         else:

   172             self.anonymous_credential = None

   173 

   174     @property

   175     def apphome(self):

   176         return self._apphome

   177     appdatahome = apphome

   178 

   179     def load_configuration(self, **kw):

   180         super(TestServerConfiguration, self).load_configuration(**kw)

   181         # no undo support in tests

   182         self.global_set_option('undo-enabled', 'n')

   183 

   184     def main_config_file(self):

   185         """return instance's control configuration file"""

   186         return join(self.apphome, '%s.conf' % self.name)

   187 

   188     def bootstrap_cubes(self):

   189         try:

   190             super(TestServerConfiguration, self).bootstrap_cubes()

   191         except IOError:

   192             # no cubes

   193             self.init_cubes( () )

   194 

   195     sourcefile = None

   196     def sources_file(self):

   197         """define in subclasses self.sourcefile if necessary"""

   198         if self.sourcefile:

   199             print('Reading sources from', self.sourcefile)

   200             sourcefile = self.sourcefile

   201             if not isabs(sourcefile):

   202                 sourcefile = join(self.apphome, sourcefile)

   203         else:

   204             sourcefile = super(TestServerConfiguration, self).sources_file()

   205         return sourcefile

   206 

   207     def read_sources_file(self):

   208         """By default, we run tests with the sqlite DB backend.  One may use its

   209         own configuration by just creating a 'sources' file in the test

   210         directory from which tests are launched or by specifying an alternative

   211         sources file using self.sourcefile.

   212         """

   213         try:

   214             sources = super(TestServerConfiguration, self).read_sources_file()

   215         except ExecutionError:

   216             sources = {}

   217         if not sources:

   218             sources = self.default_sources

   219         if 'admin' not in sources:

   220             sources['admin'] = self.default_sources['admin']

   221         return sources

   222 

   223     # web config methods needed here for cases when we use this config as a web

   224     # config

   225 

   226     def default_base_url(self):

   227         return BASE_URL

   228 

   229 

   230 class BaseApptestConfiguration(TestServerConfiguration, WebConfigurationBase):

   231     name = 'all-in-one' # so it search for all-in-one.conf, not repository.conf

   232     options = cwconfig.merge_options(TestServerConfiguration.options

   233                                      + WebConfigurationBase.options)

   234     cubicweb_appobject_path = TestServerConfiguration.cubicweb_appobject_path | WebConfigurationBase.cubicweb_appobject_path

   235     cube_appobject_path = TestServerConfiguration.cube_appobject_path | WebConfigurationBase.cube_appobject_path

   236 

   237     def available_languages(self, *args):

   238         return self.cw_languages()

   239 

   240 

   241 # XXX merge with BaseApptestConfiguration ?

   242 class ApptestConfiguration(BaseApptestConfiguration):

   243     # `skip_db_create_and_restore` controls wether or not the test database

   244     # should be created / backuped / restored. If set to True, those

   245     # steps are completely skipped, the database is used as is and is

   246     # considered initialized

   247     skip_db_create_and_restore = False

   248 

   249     def __init__(self, appid, apphome=None,

   250                  log_threshold=logging.WARNING, sourcefile=None):

   251         BaseApptestConfiguration.__init__(self, appid, apphome,

   252                                           log_threshold=log_threshold)

   253         self.init_repository = sourcefile is None

   254         self.sourcefile = sourcefile

   255 

   256 

   257 class PostgresApptestConfiguration(ApptestConfiguration):

   258     default_sources = DEFAULT_PSQL_SOURCES

   259 

   260 

   261 class RealDatabaseConfiguration(ApptestConfiguration):

   262     """configuration class for tests to run on a real database.

   263 

   264     The intialization is done by specifying a source file path.

   265 

   266     Important note: init_test_database / reset_test_database steps are

   267     skipped. It's thus up to the test developer to implement setUp/tearDown

   268     accordingly.

   269 

   270     Example usage::

   271 

   272       class MyTests(CubicWebTC):

   273           _config = RealDatabaseConfiguration('myapp',

   274                                               sourcefile='/path/to/sources')

   275 

   276           def test_something(self):

   277               with self.admin_access.web_request() as req:

   278                   rset = req.execute('Any X WHERE X is CWUser')

   279                   self.view('foaf', rset, req=req)

   280 

   281     """

   282     skip_db_create_and_restore = True

   283     read_instance_schema = True # read schema from database

   284 

   285 # test database handling #######################################################

   286 

   287 DEFAULT_EMPTY_DB_ID = '__default_empty_db__'

   288 

   289 class TestDataBaseHandler(object):

   290     DRIVER = None

   291 

   292     db_cache = {}

   293     explored_glob = set()

   294 

   295     def __init__(self, config, init_config=None):

   296         self.config = config

   297         self.init_config = init_config

   298         self._repo = None

   299         # pure consistency check

   300         assert self.system_source['db-driver'] == self.DRIVER

   301 

   302         # some handlers want to store info here, avoid a warning

   303         from cubicweb.server.sources.native import NativeSQLSource

   304         NativeSQLSource.options += (

   305             ('global-db-name',

   306              {'type': 'string', 'help': 'for internal use only'

   307             }),

   308         )

   309 

   310     def _ensure_test_backup_db_dir(self):

   311         """Return path of directory for database backup.

   312 

   313         The function create it if necessary"""

   314         backupdir = join(self.config.apphome, 'database')

   315         try:

   316             os.makedirs(backupdir)

   317         except:

   318             if not isdir(backupdir):

   319                 raise

   320         return backupdir

   321 

   322     def config_path(self, db_id):

   323         """Path for config backup of a given database id"""

   324         return self.absolute_backup_file(db_id, 'config')

   325 

   326     def absolute_backup_file(self, db_id, suffix):

   327         """Path for config backup of a given database id"""

   328         # in case db name is an absolute path, we don't want to replace anything

   329         # in parent directories

   330         directory, basename = split(self.dbname)

   331         dbname = basename.replace('-', '_')

   332         assert '.' not in db_id

   333         filename = join(directory, '%s-%s.%s' % (dbname, db_id, suffix))

   334         return join(self._ensure_test_backup_db_dir(), filename)

   335 

   336     def db_cache_key(self, db_id, dbname=None):

   337         """Build a database cache key for a db_id with the current config

   338 

   339         This key is meant to be used in the cls.db_cache mapping"""

   340         if dbname is None:

   341             dbname = self.dbname

   342         dbname = os.path.basename(dbname)

   343         dbname = dbname.replace('-', '_')

   344         return (self.config.apphome, dbname, db_id)

   345 

   346     def backup_database(self, db_id):

   347         """Store the content of the current database as <db_id>

   348 

   349         The config used are also stored."""

   350         backup_data = self._backup_database(db_id)

   351         config_path = self.config_path(db_id)

   352         # XXX we dump a dict of the config

   353         # This is an experimental to help config dependant setup (like BFSS) to

   354         # be propertly restored

   355         with tempfile.NamedTemporaryFile(dir=os.path.dirname(config_path), delete=False) as conf_file:

   356             conf_file.write(pickle.dumps(dict(self.config)))

   357         os.rename(conf_file.name, config_path)

   358         self.db_cache[self.db_cache_key(db_id)] = (backup_data, config_path)

   359 

   360     def _backup_database(self, db_id):

   361         """Actual backup the current database.

   362 

   363         return a value to be stored in db_cache to allow restoration"""

   364         raise NotImplementedError()

   365 

   366     def restore_database(self, db_id):

   367         """Restore a database.

   368 

   369         takes as argument value stored in db_cache by self._backup_database"""

   370         # XXX set a clearer error message ???

   371         backup_coordinates, config_path = self.db_cache[self.db_cache_key(db_id)]

   372         # reload the config used to create the database.

   373         with open(config_path, 'rb') as f:

   374             config = pickle.load(f)

   375         # shutdown repo before changing database content

   376         if self._repo is not None:

   377             self._repo.turn_repo_off()

   378         self._restore_database(backup_coordinates, config)

   379 

   380     def _restore_database(self, backup_coordinates, config):

   381         """Actual restore of the current database.

   382 

   383         Use the value stored in db_cache as input """

   384         raise NotImplementedError()

   385 

   386     def get_repo(self, startup=False):

   387         """ return Repository object on the current database.

   388 

   389         (turn the current repo object "on" if there is one or recreate one)

   390         if startup is True, server startup server hooks will be called if needed

   391         """

   392         if self._repo is None:

   393             self._repo = self._new_repo(self.config)

   394         # config has now been bootstrapped, call init_config if specified

   395         if self.init_config is not None:

   396             self.init_config(self.config)

   397         repo = self._repo

   398         repo.turn_repo_on()

   399         if startup and not repo._has_started:

   400             repo.hm.call_hooks('server_startup', repo=repo)

   401             repo._has_started = True

   402         return repo

   403 

   404     def _new_repo(self, config):

   405         """Factory method to create a new Repository Instance"""

   406         config._cubes = None

   407         repo = config.repository()

   408         config.repository = lambda x=None: repo

   409         # extending Repository class

   410         repo._has_started = False

   411         repo._needs_refresh = False

   412         repo.turn_repo_on = partial(turn_repo_on, repo)

   413         repo.turn_repo_off = partial(turn_repo_off, repo)

   414         return repo

   415 

   416     def get_cnx(self):

   417         """return Connection object on the current repository"""

   418         from cubicweb.repoapi import connect

   419         repo = self.get_repo()

   420         sources = self.config.read_sources_file()

   421         login  = text_type(sources['admin']['login'])

   422         password = sources['admin']['password'] or 'xxx'

   423         cnx = connect(repo, login, password=password)

   424         return cnx

   425 

   426     def get_repo_and_cnx(self, db_id=DEFAULT_EMPTY_DB_ID):

   427         """Reset database with the current db_id and return (repo, cnx)

   428 

   429         A database *MUST* have been build with the current <db_id> prior to

   430         call this method. See the ``build_db_cache`` method. The returned

   431         repository have it's startup hooks called and the connection is

   432         establised as admin."""

   433 

   434         self.restore_database(db_id)

   435         repo = self.get_repo(startup=True)

   436         cnx  = self.get_cnx()

   437         return repo, cnx

   438 

   439     @property

   440     def system_source(self):

   441         return self.config.system_source_config

   442 

   443     @property

   444     def dbname(self):

   445         return self.system_source['db-name']

   446 

   447     def init_test_database(self):

   448         """actual initialisation of the database"""

   449         raise ValueError('no initialization function for driver %r' % self.DRIVER)

   450 

   451     def has_cache(self, db_id):

   452         """Check if a given database id exist in cb cache for the current config"""

   453         cache_glob = self.absolute_backup_file('*', '*')

   454         if cache_glob not in self.explored_glob:

   455             self.discover_cached_db()

   456         return self.db_cache_key(db_id) in self.db_cache

   457 

   458     def discover_cached_db(self):

   459         """Search available db_if for the current config"""

   460         cache_glob = self.absolute_backup_file('*', '*')

   461         directory = os.path.dirname(cache_glob)

   462         entries={}

   463         candidates = glob.glob(cache_glob)

   464         for filepath in candidates:

   465             data = os.path.basename(filepath)

   466             # database backup are in the forms are <dbname>-<db_id>.<backtype>

   467             dbname, data = data.split('-', 1)

   468             db_id, filetype = data.split('.', 1)

   469             entries.setdefault((dbname, db_id), {})[filetype] = filepath

   470         for (dbname, db_id), entry in entries.items():

   471             # apply necessary transformation from the driver

   472             value = self.process_cache_entry(directory, dbname, db_id, entry)

   473             assert 'config' in entry

   474             if value is not None: # None value means "not handled by this driver

   475                                   # XXX Ignored value are shadowed to other Handler if cache are common.

   476                 key = self.db_cache_key(db_id, dbname=dbname)

   477                 self.db_cache[key] = value, entry['config']

   478         self.explored_glob.add(cache_glob)

   479 

   480     def process_cache_entry(self, directory, dbname, db_id, entry):

   481         """Transforms potential cache entry to proper backup coordinate

   482 

   483         entry argument is a "filetype" -> "filepath" mapping

   484         Return None if an entry should be ignored."""

   485         return None

   486 

   487     def build_db_cache(self, test_db_id=DEFAULT_EMPTY_DB_ID, pre_setup_func=None):

   488         """Build Database cache for ``test_db_id`` if a cache doesn't exist

   489 

   490         if ``test_db_id is DEFAULT_EMPTY_DB_ID`` self.init_test_database is

   491         called. otherwise, DEFAULT_EMPTY_DB_ID is build/restored and

   492         ``pre_setup_func`` to setup the database.

   493 

   494         This function backup any database it build"""

   495         if self.has_cache(test_db_id):

   496             return #test_db_id, 'already in cache'

   497         if test_db_id is DEFAULT_EMPTY_DB_ID:

   498             self.init_test_database()

   499         else:

   500             print('Building %s for database %s' % (test_db_id, self.dbname))

   501             self.build_db_cache(DEFAULT_EMPTY_DB_ID)

   502             self.restore_database(DEFAULT_EMPTY_DB_ID)

   503             repo = self.get_repo(startup=True)

   504             cnx = self.get_cnx()

   505             with cnx:

   506                 pre_setup_func(cnx, self.config)

   507                 cnx.commit()

   508         self.backup_database(test_db_id)

   509 

   510 

   511 class NoCreateDropDatabaseHandler(TestDataBaseHandler):

   512     """This handler is used if config.skip_db_create_and_restore is True

   513 

   514     This is typically the case with RealDBConfig. In that case,

   515     we explicitely want to skip init / backup / restore phases.

   516 

   517     This handler redefines the three corresponding methods and delegates

   518     to original handler for any other method / attribute

   519     """

   520 

   521     def __init__(self, base_handler):

   522         self.base_handler = base_handler

   523 

   524     # override init / backup / restore methods

   525     def init_test_database(self):

   526         pass

   527 

   528     def backup_database(self, db_id):

   529         pass

   530 

   531     def restore_database(self, db_id):

   532         pass

   533 

   534     # delegate to original handler in all other cases

   535     def __getattr__(self, attrname):

   536         return getattr(self.base_handler, attrname)

   537 

   538 

   539 ### postgres test database handling ############################################

   540 

   541 def startpgcluster(pyfile):

   542     """Start a postgresql cluster next to pyfile"""

   543     datadir = join(os.path.dirname(pyfile), 'data', 'database',

   544                    'pgdb-%s' % os.path.splitext(os.path.basename(pyfile))[0])

   545     if not exists(datadir):

   546         try:

   547             subprocess.check_call(['initdb', '-D', datadir, '-E', 'utf-8', '--locale=C'])

   548 

   549         except OSError as err:

   550             if err.errno == errno.ENOENT:

   551                 raise OSError('"initdb" could not be found. '

   552                               'You should add the postgresql bin folder to your PATH '

   553                               '(/usr/lib/postgresql/9.1/bin for example).')

   554             raise

   555     datadir = os.path.abspath(datadir)

   556     pgport = '5432'

   557     env = os.environ.copy()

   558     sockdir = tempfile.mkdtemp(prefix='cwpg')

   559     DEFAULT_PSQL_SOURCES['system']['db-host'] = sockdir

   560     DEFAULT_PSQL_SOURCES['system']['db-port'] = pgport

   561     options = '-h "" -k %s -p %s' % (sockdir, pgport)

   562     options += ' -c fsync=off -c full_page_writes=off'

   563     options += ' -c synchronous_commit=off'

   564     try:

   565         subprocess.check_call(['pg_ctl', 'start', '-w', '-D', datadir,

   566                                '-o', options],

   567                               env=env)

   568     except OSError as err:

   569         try:

   570             os.rmdir(sockdir)

   571         except OSError:

   572             pass

   573         if err.errno == errno.ENOENT:

   574             raise OSError('"pg_ctl" could not be found. '

   575                           'You should add the postgresql bin folder to your PATH '

   576                           '(/usr/lib/postgresql/9.1/bin for example).')

   577         raise

   578 

   579 

   580 def stoppgcluster(pyfile):

   581     """Kill the postgresql cluster running next to pyfile"""

   582     datadir = join(os.path.dirname(pyfile), 'data', 'database',

   583                    'pgdb-%s' % os.path.splitext(os.path.basename(pyfile))[0])

   584     subprocess.call(['pg_ctl', 'stop', '-D', datadir, '-m', 'fast'])

   585     try:

   586         os.rmdir(DEFAULT_PSQL_SOURCES['system']['db-host'])

   587     except OSError:

   588         pass

   589 

   590 

   591 class PostgresTestDataBaseHandler(TestDataBaseHandler):

   592     DRIVER = 'postgres'

   593 

   594     # Separate db_cache for PG databases, to avoid collisions with sqlite dbs

   595     db_cache = {}

   596     explored_glob = set()

   597 

   598     __CTL = set()

   599 

   600     def __init__(self, *args, **kwargs):

   601         super(PostgresTestDataBaseHandler, self).__init__(*args, **kwargs)

   602         if 'global-db-name' not in self.system_source:

   603             self.system_source['global-db-name'] = self.system_source['db-name']

   604             self.system_source['db-name'] = self.system_source['db-name'] + str(os.getpid())

   605 

   606     @property

   607     @cached

   608     def helper(self):

   609         from logilab.database import get_db_helper

   610         return get_db_helper('postgres')

   611 

   612     @property

   613     def dbname(self):

   614         return self.system_source['global-db-name']

   615 

   616     @property

   617     def dbcnx(self):

   618         try:

   619             return self._cnx

   620         except AttributeError:

   621             from cubicweb.server.serverctl import _db_sys_cnx

   622             try:

   623                 self._cnx = _db_sys_cnx(

   624                     self.system_source, 'CREATE DATABASE and / or USER',

   625                     interactive=False)

   626                 return self._cnx

   627             except Exception:

   628                 self._cnx = None

   629                 raise

   630 

   631     @property

   632     @cached

   633     def cursor(self):

   634         return self.dbcnx.cursor()

   635 

   636     def process_cache_entry(self, directory, dbname, db_id, entry):

   637         backup_name = self._backup_name(db_id)

   638         if backup_name in self.helper.list_databases(self.cursor):

   639             return backup_name

   640         return None

   641 

   642     def has_cache(self, db_id):

   643         backup_name = self._backup_name(db_id)

   644         return (super(PostgresTestDataBaseHandler, self).has_cache(db_id)

   645                 and backup_name in self.helper.list_databases(self.cursor))

   646 

   647     def init_test_database(self):

   648         """initialize a fresh postgresql database used for testing purpose"""

   649         from cubicweb.server import init_repository

   650         from cubicweb.server.serverctl import system_source_cnx, createdb

   651         # connect on the dbms system base to create our base

   652         try:

   653             self._drop(self.system_source['db-name'])

   654             createdb(self.helper, self.system_source, self.dbcnx, self.cursor)

   655             self.dbcnx.commit()

   656             cnx = system_source_cnx(self.system_source, special_privs='LANGUAGE C',

   657                                     interactive=False)

   658             templcursor = cnx.cursor()

   659             try:

   660                 # XXX factorize with db-create code

   661                 self.helper.init_fti_extensions(templcursor)

   662                 # install plpythonu/plpgsql language if not installed by the cube

   663                 langs = sys.platform == 'win32' and ('plpgsql',) or ('plpythonu', 'plpgsql')

   664                 for extlang in langs:

   665                     self.helper.create_language(templcursor, extlang)

   666                 cnx.commit()

   667             finally:

   668                 templcursor.close()

   669                 cnx.close()

   670             init_repository(self.config, interactive=False,

   671                             init_config=self.init_config)

   672         except BaseException:

   673             if self.dbcnx is not None:

   674                 self.dbcnx.rollback()

   675             sys.stderr.write('building %s failed\n' % self.dbname)

   676             #self._drop(self.dbname)

   677             raise

   678 

   679     def helper_clear_cache(self):

   680         if self.dbcnx is not None:

   681             self.dbcnx.commit()

   682             self.dbcnx.close()

   683             del self._cnx

   684             clear_cache(self, 'cursor')

   685         clear_cache(self, 'helper')

   686 

   687     def __del__(self):

   688         self.helper_clear_cache()

   689 

   690     @property

   691     def _config_id(self):

   692         return sha1(self.config.apphome.encode('utf-8')).hexdigest()[:10]

   693 

   694     def _backup_name(self, db_id): # merge me with parent

   695         backup_name = '_'.join(('cache', self._config_id, self.dbname, db_id))

   696         return backup_name.lower()

   697 

   698     def _drop(self, db_name):

   699         if db_name in self.helper.list_databases(self.cursor):

   700             self.cursor.execute('DROP DATABASE %s' % db_name)

   701             self.dbcnx.commit()

   702 

   703     def _backup_database(self, db_id):

   704         """Actual backup the current database.

   705 

   706         return a value to be stored in db_cache to allow restoration

   707         """

   708         from cubicweb.server.serverctl import createdb

   709         orig_name = self.system_source['db-name']

   710         try:

   711             backup_name = self._backup_name(db_id)

   712             self._drop(backup_name)

   713             self.system_source['db-name'] = backup_name

   714             if self._repo:

   715                 self._repo.turn_repo_off()

   716             try:

   717                 createdb(self.helper, self.system_source, self.dbcnx, self.cursor, template=orig_name)

   718                 self.dbcnx.commit()

   719             finally:

   720                 if self._repo:

   721                     self._repo.turn_repo_on()

   722             return backup_name

   723         finally:

   724             self.system_source['db-name'] = orig_name

   725 

   726     def _restore_database(self, backup_coordinates, config):

   727         from cubicweb.server.serverctl import createdb

   728         """Actual restore of the current database.

   729 

   730         Use the value tostored in db_cache as input """

   731         self._drop(self.system_source['db-name'])

   732         createdb(self.helper, self.system_source, self.dbcnx, self.cursor,

   733                  template=backup_coordinates)

   734         self.dbcnx.commit()

   735 

   736 

   737 

   738 ### sqlserver2005 test database handling #######################################

   739 

   740 class SQLServerTestDataBaseHandler(TestDataBaseHandler):

   741     DRIVER = 'sqlserver'

   742 

   743     # XXX complete me

   744 

   745     def init_test_database(self):

   746         """initialize a fresh sqlserver databse used for testing purpose"""

   747         if self.config.init_repository:

   748             from cubicweb.server import init_repository

   749             init_repository(self.config, interactive=False, drop=True,

   750                             init_config=self.init_config)

   751 

   752 ### sqlite test database handling ##############################################

   753 

   754 class SQLiteTestDataBaseHandler(TestDataBaseHandler):

   755     DRIVER = 'sqlite'

   756 

   757     __TMPDB = set()

   758 

   759     @classmethod

   760     def _cleanup_all_tmpdb(cls):

   761         for dbpath in cls.__TMPDB:

   762             cls._cleanup_database(dbpath)

   763 

   764 

   765 

   766     def __init__(self, *args, **kwargs):

   767         super(SQLiteTestDataBaseHandler, self).__init__(*args, **kwargs)

   768         # use a dedicated base for each process.

   769         if 'global-db-name' not in self.system_source:

   770             self.system_source['global-db-name'] = self.system_source['db-name']

   771             process_db = self.system_source['db-name'] + str(os.getpid())

   772             self.system_source['db-name'] = process_db

   773         process_db = self.absolute_dbfile() # update db-name to absolute path

   774         self.__TMPDB.add(process_db)

   775 

   776     @staticmethod

   777     def _cleanup_database(dbfile):

   778         try:

   779             os.remove(dbfile)

   780             os.remove('%s-journal' % dbfile)

   781         except OSError:

   782             pass

   783 

   784     @property

   785     def dbname(self):

   786         return self.system_source['global-db-name']

   787 

   788     def absolute_dbfile(self):

   789         """absolute path of current database file"""

   790         dbfile = join(self._ensure_test_backup_db_dir(),

   791                       self.system_source['db-name'])

   792         self.system_source['db-name'] = dbfile

   793         return dbfile

   794 

   795     def process_cache_entry(self, directory, dbname, db_id, entry):

   796         return entry.get('sqlite')

   797 

   798     def _backup_database(self, db_id=DEFAULT_EMPTY_DB_ID):

   799         # XXX remove database file if it exists ???

   800         dbfile = self.absolute_dbfile()

   801         backup_file = self.absolute_backup_file(db_id, 'sqlite')

   802         shutil.copy(dbfile, backup_file)

   803         # Useful to debug WHO writes a database

   804         # backup_stack = self.absolute_backup_file(db_id, '.stack')

   805         #with open(backup_stack, 'w') as backup_stack_file:

   806         #    import traceback

   807         #    traceback.print_stack(file=backup_stack_file)

   808         return backup_file

   809 

   810     def _restore_database(self, backup_coordinates, _config):

   811         # remove database file if it exists ?

   812         dbfile = self.absolute_dbfile()

   813         self._cleanup_database(dbfile)

   814         shutil.copy(backup_coordinates, dbfile)

   815         self.get_repo()

   816 

   817     def init_test_database(self):

   818         """initialize a fresh sqlite databse used for testing purpose"""

   819         # initialize the database

   820         from cubicweb.server import init_repository

   821         self._cleanup_database(self.absolute_dbfile())

   822         init_repository(self.config, interactive=False,

   823                         init_config=self.init_config)

   824 

   825 import atexit

   826 atexit.register(SQLiteTestDataBaseHandler._cleanup_all_tmpdb)

   827 

   828 

   829 HANDLERS = {}

   830 

   831 def register_handler(handlerkls, overwrite=False):

   832     assert handlerkls is not None

   833     if overwrite or handlerkls.DRIVER not in HANDLERS:

   834         HANDLERS[handlerkls.DRIVER] = handlerkls

   835     else:

   836         msg = "%s: Handler already exists use overwrite if it's intended\n"\

   837               "(existing handler class is %r)"

   838         raise ValueError(msg % (handlerkls.DRIVER, HANDLERS[handlerkls.DRIVER]))

   839 

   840 register_handler(PostgresTestDataBaseHandler)

   841 register_handler(SQLiteTestDataBaseHandler)

   842 register_handler(SQLServerTestDataBaseHandler)

   843 

   844 

   845 class HCache(object):

   846     """Handler cache object: store database handler for a given configuration.

   847 

   848     We only keep one repo in cache to prevent too much objects to stay alive

   849     (database handler holds a reference to a repository). As at the moment a new

   850     handler is created for each TestCase class and all test methods are executed

   851     sequentially whithin this class, there should not have more cache miss that

   852     if we had a wider cache as once a Handler stop being used it won't be used

   853     again.

   854     """

   855 

   856     def __init__(self):

   857         self.config = None

   858         self.handler = None

   859 

   860     def get(self, config):

   861         if config is self.config:

   862             return self.handler

   863         else:

   864             return None

   865 

   866     def set(self, config, handler):

   867         self.config = config

   868         self.handler = handler

   869 

   870 HCACHE = HCache()

   871 

   872 

   873 # XXX a class method on Test ?

   874 

   875 _CONFIG = None

   876 def get_test_db_handler(config, init_config=None):

   877     global _CONFIG

   878     if _CONFIG is not None and config is not _CONFIG:

   879         from logilab.common.modutils import cleanup_sys_modules

   880         # cleanup all dynamically loaded modules and everything in the instance

   881         # directory

   882         apphome = _CONFIG.apphome

   883         if apphome: # may be unset in tests

   884             cleanup_sys_modules([apphome])

   885         # also cleanup sys.path

   886         if apphome in sys.path:

   887             sys.path.remove(apphome)

   888     _CONFIG = config

   889     config.adjust_sys_path()

   890     handler = HCACHE.get(config)

   891     if handler is not None:

   892         return handler

   893     driver = config.system_source_config['db-driver']

   894     key = (driver, config)

   895     handlerkls = HANDLERS.get(driver, None)

   896     if handlerkls is not None:

   897         handler = handlerkls(config, init_config)

   898         if config.skip_db_create_and_restore:

   899             handler = NoCreateDropDatabaseHandler(handler)

   900         HCACHE.set(config, handler)

   901         return handler

   902     else:

   903         raise ValueError('no initialization function for driver %r' % driver)

   904 

   905 ### compatibility layer ##############################################

   906 from logilab.common.deprecation import deprecated

   907 

   908 @deprecated("please use the new DatabaseHandler mecanism")

   909 def init_test_database(config=None, configdir='data', apphome=None):

   910     """init a test database for a specific driver"""

   911     if config is None:

   912         config = TestServerConfiguration(apphome=apphome)

   913     handler = get_test_db_handler(config)

   914     handler.build_db_cache()

   915     return handler.get_repo_and_cnx()