1 # copyright 2003-2015 LOGILAB S.A. (Paris, FRANCE), all rights reserved. |
|
2 # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr |
|
3 # |
|
4 # This file is part of CubicWeb. |
|
5 # |
|
6 # CubicWeb is free software: you can redistribute it and/or modify it under the |
|
7 # terms of the GNU Lesser General Public License as published by the Free |
|
8 # Software Foundation, either version 2.1 of the License, or (at your option) |
|
9 # any later version. |
|
10 # |
|
11 # CubicWeb is distributed in the hope that it will be useful, but WITHOUT |
|
12 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS |
|
13 # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more |
|
14 # details. |
|
15 # |
|
16 # You should have received a copy of the GNU Lesser General Public License along |
|
17 # with CubicWeb. If not, see <http://www.gnu.org/licenses/>. |
|
18 """Test tools for cubicweb""" |
|
19 from __future__ import print_function |
|
20 |
|
21 __docformat__ = "restructuredtext en" |
|
22 |
|
23 import os |
|
24 import sys |
|
25 import errno |
|
26 import logging |
|
27 import shutil |
|
28 import glob |
|
29 import subprocess |
|
30 import warnings |
|
31 import tempfile |
|
32 import getpass |
|
33 from hashlib import sha1 # pylint: disable=E0611 |
|
34 from datetime import timedelta |
|
35 from os.path import abspath, join, exists, split, isabs, isdir |
|
36 from functools import partial |
|
37 |
|
38 from six import text_type |
|
39 from six.moves import cPickle as pickle |
|
40 |
|
41 from logilab.common.date import strptime |
|
42 from logilab.common.decorators import cached, clear_cache |
|
43 |
|
44 from cubicweb import ExecutionError, BadConnectionId |
|
45 from cubicweb import schema, cwconfig |
|
46 from cubicweb.server.serverconfig import ServerConfiguration |
|
47 from cubicweb.etwist.twconfig import WebConfigurationBase |
|
48 |
|
49 cwconfig.CubicWebConfiguration.cls_adjust_sys_path() |
|
50 |
|
51 # db auto-population configuration ############################################# |
|
52 |
|
53 SYSTEM_ENTITIES = (schema.SCHEMA_TYPES |
|
54 | schema.INTERNAL_TYPES |
|
55 | schema.WORKFLOW_TYPES |
|
56 | set(('CWGroup', 'CWUser',)) |
|
57 ) |
|
58 SYSTEM_RELATIONS = (schema.META_RTYPES |
|
59 | schema.WORKFLOW_RTYPES |
|
60 | schema.WORKFLOW_DEF_RTYPES |
|
61 | schema.SYSTEM_RTYPES |
|
62 | schema.SCHEMA_TYPES |
|
63 | set(('primary_email', # deducted from other relations |
|
64 )) |
|
65 ) |
|
66 |
|
67 # content validation configuration ############################################# |
|
68 |
|
69 # validators are used to validate (XML, DTD, whatever) view's content |
|
70 # validators availables are : |
|
71 # 'dtd' : validates XML + declared DTD |
|
72 # 'xml' : guarantees XML is well formed |
|
73 # None : do not try to validate anything |
|
74 |
|
75 # {'vid': validator} |
|
76 VIEW_VALIDATORS = {} |
|
77 |
|
78 |
|
79 # cubicweb test configuration ################################################## |
|
80 |
|
81 BASE_URL = 'http://testing.fr/cubicweb/' |
|
82 |
|
83 DEFAULT_SOURCES = {'system': {'adapter' : 'native', |
|
84 'db-encoding' : 'UTF-8', #'ISO-8859-1', |
|
85 'db-user' : u'admin', |
|
86 'db-password' : 'gingkow', |
|
87 'db-name' : 'tmpdb', |
|
88 'db-driver' : 'sqlite', |
|
89 'db-host' : None, |
|
90 }, |
|
91 'admin' : {'login': u'admin', |
|
92 'password': u'gingkow', |
|
93 }, |
|
94 } |
|
95 DEFAULT_PSQL_SOURCES = DEFAULT_SOURCES.copy() |
|
96 DEFAULT_PSQL_SOURCES['system'] = DEFAULT_SOURCES['system'].copy() |
|
97 DEFAULT_PSQL_SOURCES['system']['db-driver'] = 'postgres' |
|
98 DEFAULT_PSQL_SOURCES['system']['db-user'] = text_type(getpass.getuser()) |
|
99 DEFAULT_PSQL_SOURCES['system']['db-password'] = None |
|
100 |
|
101 def turn_repo_off(repo): |
|
102 """ Idea: this is less costly than a full re-creation of the repo object. |
|
103 off: |
|
104 * session are closed, |
|
105 * cnxsets are closed |
|
106 * system source is shutdown |
|
107 """ |
|
108 if not repo._needs_refresh: |
|
109 for sessionid in list(repo._sessions): |
|
110 warnings.warn('%s Open session found while turning repository off' |
|
111 %sessionid, RuntimeWarning) |
|
112 try: |
|
113 repo.close(sessionid) |
|
114 except BadConnectionId: #this is strange ? thread issue ? |
|
115 print('XXX unknown session', sessionid) |
|
116 for cnxset in repo.cnxsets: |
|
117 cnxset.close(True) |
|
118 repo.system_source.shutdown() |
|
119 repo._needs_refresh = True |
|
120 repo._has_started = False |
|
121 |
|
122 |
|
123 def turn_repo_on(repo): |
|
124 """Idea: this is less costly than a full re-creation of the repo object. |
|
125 on: |
|
126 * cnxsets are connected |
|
127 * cache are cleared |
|
128 """ |
|
129 if repo._needs_refresh: |
|
130 for cnxset in repo.cnxsets: |
|
131 cnxset.reconnect() |
|
132 repo._type_source_cache = {} |
|
133 repo._extid_cache = {} |
|
134 repo.querier._rql_cache = {} |
|
135 repo.system_source.reset_caches() |
|
136 repo._needs_refresh = False |
|
137 |
|
138 |
|
139 class TestServerConfiguration(ServerConfiguration): |
|
140 mode = 'test' |
|
141 read_instance_schema = False |
|
142 init_repository = True |
|
143 skip_db_create_and_restore = False |
|
144 default_sources = DEFAULT_SOURCES |
|
145 |
|
146 def __init__(self, appid='data', apphome=None, log_threshold=logging.CRITICAL+10): |
|
147 # must be set before calling parent __init__ |
|
148 if apphome is None: |
|
149 if exists(appid): |
|
150 apphome = abspath(appid) |
|
151 else: # cube test |
|
152 apphome = abspath('..') |
|
153 self._apphome = apphome |
|
154 super(TestServerConfiguration, self).__init__(appid) |
|
155 self.init_log(log_threshold, force=True) |
|
156 # need this, usually triggered by cubicweb-ctl |
|
157 self.load_cwctl_plugins() |
|
158 |
|
159 # By default anonymous login are allow but some test need to deny of to |
|
160 # change the default user. Set it to None to prevent anonymous login. |
|
161 anonymous_credential = ('anon', 'anon') |
|
162 |
|
163 def anonymous_user(self): |
|
164 if not self.anonymous_credential: |
|
165 return None, None |
|
166 return self.anonymous_credential |
|
167 |
|
168 def set_anonymous_allowed(self, allowed, anonuser=u'anon'): |
|
169 if allowed: |
|
170 self.anonymous_credential = (anonuser, anonuser) |
|
171 else: |
|
172 self.anonymous_credential = None |
|
173 |
|
174 @property |
|
175 def apphome(self): |
|
176 return self._apphome |
|
177 appdatahome = apphome |
|
178 |
|
179 def load_configuration(self, **kw): |
|
180 super(TestServerConfiguration, self).load_configuration(**kw) |
|
181 # no undo support in tests |
|
182 self.global_set_option('undo-enabled', 'n') |
|
183 |
|
184 def main_config_file(self): |
|
185 """return instance's control configuration file""" |
|
186 return join(self.apphome, '%s.conf' % self.name) |
|
187 |
|
188 def bootstrap_cubes(self): |
|
189 try: |
|
190 super(TestServerConfiguration, self).bootstrap_cubes() |
|
191 except IOError: |
|
192 # no cubes |
|
193 self.init_cubes( () ) |
|
194 |
|
195 sourcefile = None |
|
196 def sources_file(self): |
|
197 """define in subclasses self.sourcefile if necessary""" |
|
198 if self.sourcefile: |
|
199 print('Reading sources from', self.sourcefile) |
|
200 sourcefile = self.sourcefile |
|
201 if not isabs(sourcefile): |
|
202 sourcefile = join(self.apphome, sourcefile) |
|
203 else: |
|
204 sourcefile = super(TestServerConfiguration, self).sources_file() |
|
205 return sourcefile |
|
206 |
|
207 def read_sources_file(self): |
|
208 """By default, we run tests with the sqlite DB backend. One may use its |
|
209 own configuration by just creating a 'sources' file in the test |
|
210 directory from which tests are launched or by specifying an alternative |
|
211 sources file using self.sourcefile. |
|
212 """ |
|
213 try: |
|
214 sources = super(TestServerConfiguration, self).read_sources_file() |
|
215 except ExecutionError: |
|
216 sources = {} |
|
217 if not sources: |
|
218 sources = self.default_sources |
|
219 if 'admin' not in sources: |
|
220 sources['admin'] = self.default_sources['admin'] |
|
221 return sources |
|
222 |
|
223 # web config methods needed here for cases when we use this config as a web |
|
224 # config |
|
225 |
|
226 def default_base_url(self): |
|
227 return BASE_URL |
|
228 |
|
229 |
|
230 class BaseApptestConfiguration(TestServerConfiguration, WebConfigurationBase): |
|
231 name = 'all-in-one' # so it search for all-in-one.conf, not repository.conf |
|
232 options = cwconfig.merge_options(TestServerConfiguration.options |
|
233 + WebConfigurationBase.options) |
|
234 cubicweb_appobject_path = TestServerConfiguration.cubicweb_appobject_path | WebConfigurationBase.cubicweb_appobject_path |
|
235 cube_appobject_path = TestServerConfiguration.cube_appobject_path | WebConfigurationBase.cube_appobject_path |
|
236 |
|
237 def available_languages(self, *args): |
|
238 return self.cw_languages() |
|
239 |
|
240 |
|
241 # XXX merge with BaseApptestConfiguration ? |
|
242 class ApptestConfiguration(BaseApptestConfiguration): |
|
243 # `skip_db_create_and_restore` controls wether or not the test database |
|
244 # should be created / backuped / restored. If set to True, those |
|
245 # steps are completely skipped, the database is used as is and is |
|
246 # considered initialized |
|
247 skip_db_create_and_restore = False |
|
248 |
|
249 def __init__(self, appid, apphome=None, |
|
250 log_threshold=logging.WARNING, sourcefile=None): |
|
251 BaseApptestConfiguration.__init__(self, appid, apphome, |
|
252 log_threshold=log_threshold) |
|
253 self.init_repository = sourcefile is None |
|
254 self.sourcefile = sourcefile |
|
255 |
|
256 |
|
257 class PostgresApptestConfiguration(ApptestConfiguration): |
|
258 default_sources = DEFAULT_PSQL_SOURCES |
|
259 |
|
260 |
|
261 class RealDatabaseConfiguration(ApptestConfiguration): |
|
262 """configuration class for tests to run on a real database. |
|
263 |
|
264 The intialization is done by specifying a source file path. |
|
265 |
|
266 Important note: init_test_database / reset_test_database steps are |
|
267 skipped. It's thus up to the test developer to implement setUp/tearDown |
|
268 accordingly. |
|
269 |
|
270 Example usage:: |
|
271 |
|
272 class MyTests(CubicWebTC): |
|
273 _config = RealDatabaseConfiguration('myapp', |
|
274 sourcefile='/path/to/sources') |
|
275 |
|
276 def test_something(self): |
|
277 with self.admin_access.web_request() as req: |
|
278 rset = req.execute('Any X WHERE X is CWUser') |
|
279 self.view('foaf', rset, req=req) |
|
280 |
|
281 """ |
|
282 skip_db_create_and_restore = True |
|
283 read_instance_schema = True # read schema from database |
|
284 |
|
285 # test database handling ####################################################### |
|
286 |
|
287 DEFAULT_EMPTY_DB_ID = '__default_empty_db__' |
|
288 |
|
289 class TestDataBaseHandler(object): |
|
290 DRIVER = None |
|
291 |
|
292 db_cache = {} |
|
293 explored_glob = set() |
|
294 |
|
295 def __init__(self, config, init_config=None): |
|
296 self.config = config |
|
297 self.init_config = init_config |
|
298 self._repo = None |
|
299 # pure consistency check |
|
300 assert self.system_source['db-driver'] == self.DRIVER |
|
301 |
|
302 # some handlers want to store info here, avoid a warning |
|
303 from cubicweb.server.sources.native import NativeSQLSource |
|
304 NativeSQLSource.options += ( |
|
305 ('global-db-name', |
|
306 {'type': 'string', 'help': 'for internal use only' |
|
307 }), |
|
308 ) |
|
309 |
|
310 def _ensure_test_backup_db_dir(self): |
|
311 """Return path of directory for database backup. |
|
312 |
|
313 The function create it if necessary""" |
|
314 backupdir = join(self.config.apphome, 'database') |
|
315 try: |
|
316 os.makedirs(backupdir) |
|
317 except: |
|
318 if not isdir(backupdir): |
|
319 raise |
|
320 return backupdir |
|
321 |
|
322 def config_path(self, db_id): |
|
323 """Path for config backup of a given database id""" |
|
324 return self.absolute_backup_file(db_id, 'config') |
|
325 |
|
326 def absolute_backup_file(self, db_id, suffix): |
|
327 """Path for config backup of a given database id""" |
|
328 # in case db name is an absolute path, we don't want to replace anything |
|
329 # in parent directories |
|
330 directory, basename = split(self.dbname) |
|
331 dbname = basename.replace('-', '_') |
|
332 assert '.' not in db_id |
|
333 filename = join(directory, '%s-%s.%s' % (dbname, db_id, suffix)) |
|
334 return join(self._ensure_test_backup_db_dir(), filename) |
|
335 |
|
336 def db_cache_key(self, db_id, dbname=None): |
|
337 """Build a database cache key for a db_id with the current config |
|
338 |
|
339 This key is meant to be used in the cls.db_cache mapping""" |
|
340 if dbname is None: |
|
341 dbname = self.dbname |
|
342 dbname = os.path.basename(dbname) |
|
343 dbname = dbname.replace('-', '_') |
|
344 return (self.config.apphome, dbname, db_id) |
|
345 |
|
346 def backup_database(self, db_id): |
|
347 """Store the content of the current database as <db_id> |
|
348 |
|
349 The config used are also stored.""" |
|
350 backup_data = self._backup_database(db_id) |
|
351 config_path = self.config_path(db_id) |
|
352 # XXX we dump a dict of the config |
|
353 # This is an experimental to help config dependant setup (like BFSS) to |
|
354 # be propertly restored |
|
355 with tempfile.NamedTemporaryFile(dir=os.path.dirname(config_path), delete=False) as conf_file: |
|
356 conf_file.write(pickle.dumps(dict(self.config))) |
|
357 os.rename(conf_file.name, config_path) |
|
358 self.db_cache[self.db_cache_key(db_id)] = (backup_data, config_path) |
|
359 |
|
360 def _backup_database(self, db_id): |
|
361 """Actual backup the current database. |
|
362 |
|
363 return a value to be stored in db_cache to allow restoration""" |
|
364 raise NotImplementedError() |
|
365 |
|
366 def restore_database(self, db_id): |
|
367 """Restore a database. |
|
368 |
|
369 takes as argument value stored in db_cache by self._backup_database""" |
|
370 # XXX set a clearer error message ??? |
|
371 backup_coordinates, config_path = self.db_cache[self.db_cache_key(db_id)] |
|
372 # reload the config used to create the database. |
|
373 with open(config_path, 'rb') as f: |
|
374 config = pickle.load(f) |
|
375 # shutdown repo before changing database content |
|
376 if self._repo is not None: |
|
377 self._repo.turn_repo_off() |
|
378 self._restore_database(backup_coordinates, config) |
|
379 |
|
380 def _restore_database(self, backup_coordinates, config): |
|
381 """Actual restore of the current database. |
|
382 |
|
383 Use the value stored in db_cache as input """ |
|
384 raise NotImplementedError() |
|
385 |
|
386 def get_repo(self, startup=False): |
|
387 """ return Repository object on the current database. |
|
388 |
|
389 (turn the current repo object "on" if there is one or recreate one) |
|
390 if startup is True, server startup server hooks will be called if needed |
|
391 """ |
|
392 if self._repo is None: |
|
393 self._repo = self._new_repo(self.config) |
|
394 # config has now been bootstrapped, call init_config if specified |
|
395 if self.init_config is not None: |
|
396 self.init_config(self.config) |
|
397 repo = self._repo |
|
398 repo.turn_repo_on() |
|
399 if startup and not repo._has_started: |
|
400 repo.hm.call_hooks('server_startup', repo=repo) |
|
401 repo._has_started = True |
|
402 return repo |
|
403 |
|
404 def _new_repo(self, config): |
|
405 """Factory method to create a new Repository Instance""" |
|
406 config._cubes = None |
|
407 repo = config.repository() |
|
408 config.repository = lambda x=None: repo |
|
409 # extending Repository class |
|
410 repo._has_started = False |
|
411 repo._needs_refresh = False |
|
412 repo.turn_repo_on = partial(turn_repo_on, repo) |
|
413 repo.turn_repo_off = partial(turn_repo_off, repo) |
|
414 return repo |
|
415 |
|
416 def get_cnx(self): |
|
417 """return Connection object on the current repository""" |
|
418 from cubicweb.repoapi import connect |
|
419 repo = self.get_repo() |
|
420 sources = self.config.read_sources_file() |
|
421 login = text_type(sources['admin']['login']) |
|
422 password = sources['admin']['password'] or 'xxx' |
|
423 cnx = connect(repo, login, password=password) |
|
424 return cnx |
|
425 |
|
426 def get_repo_and_cnx(self, db_id=DEFAULT_EMPTY_DB_ID): |
|
427 """Reset database with the current db_id and return (repo, cnx) |
|
428 |
|
429 A database *MUST* have been build with the current <db_id> prior to |
|
430 call this method. See the ``build_db_cache`` method. The returned |
|
431 repository have it's startup hooks called and the connection is |
|
432 establised as admin.""" |
|
433 |
|
434 self.restore_database(db_id) |
|
435 repo = self.get_repo(startup=True) |
|
436 cnx = self.get_cnx() |
|
437 return repo, cnx |
|
438 |
|
439 @property |
|
440 def system_source(self): |
|
441 return self.config.system_source_config |
|
442 |
|
443 @property |
|
444 def dbname(self): |
|
445 return self.system_source['db-name'] |
|
446 |
|
447 def init_test_database(self): |
|
448 """actual initialisation of the database""" |
|
449 raise ValueError('no initialization function for driver %r' % self.DRIVER) |
|
450 |
|
451 def has_cache(self, db_id): |
|
452 """Check if a given database id exist in cb cache for the current config""" |
|
453 cache_glob = self.absolute_backup_file('*', '*') |
|
454 if cache_glob not in self.explored_glob: |
|
455 self.discover_cached_db() |
|
456 return self.db_cache_key(db_id) in self.db_cache |
|
457 |
|
458 def discover_cached_db(self): |
|
459 """Search available db_if for the current config""" |
|
460 cache_glob = self.absolute_backup_file('*', '*') |
|
461 directory = os.path.dirname(cache_glob) |
|
462 entries={} |
|
463 candidates = glob.glob(cache_glob) |
|
464 for filepath in candidates: |
|
465 data = os.path.basename(filepath) |
|
466 # database backup are in the forms are <dbname>-<db_id>.<backtype> |
|
467 dbname, data = data.split('-', 1) |
|
468 db_id, filetype = data.split('.', 1) |
|
469 entries.setdefault((dbname, db_id), {})[filetype] = filepath |
|
470 for (dbname, db_id), entry in entries.items(): |
|
471 # apply necessary transformation from the driver |
|
472 value = self.process_cache_entry(directory, dbname, db_id, entry) |
|
473 assert 'config' in entry |
|
474 if value is not None: # None value means "not handled by this driver |
|
475 # XXX Ignored value are shadowed to other Handler if cache are common. |
|
476 key = self.db_cache_key(db_id, dbname=dbname) |
|
477 self.db_cache[key] = value, entry['config'] |
|
478 self.explored_glob.add(cache_glob) |
|
479 |
|
480 def process_cache_entry(self, directory, dbname, db_id, entry): |
|
481 """Transforms potential cache entry to proper backup coordinate |
|
482 |
|
483 entry argument is a "filetype" -> "filepath" mapping |
|
484 Return None if an entry should be ignored.""" |
|
485 return None |
|
486 |
|
487 def build_db_cache(self, test_db_id=DEFAULT_EMPTY_DB_ID, pre_setup_func=None): |
|
488 """Build Database cache for ``test_db_id`` if a cache doesn't exist |
|
489 |
|
490 if ``test_db_id is DEFAULT_EMPTY_DB_ID`` self.init_test_database is |
|
491 called. otherwise, DEFAULT_EMPTY_DB_ID is build/restored and |
|
492 ``pre_setup_func`` to setup the database. |
|
493 |
|
494 This function backup any database it build""" |
|
495 if self.has_cache(test_db_id): |
|
496 return #test_db_id, 'already in cache' |
|
497 if test_db_id is DEFAULT_EMPTY_DB_ID: |
|
498 self.init_test_database() |
|
499 else: |
|
500 print('Building %s for database %s' % (test_db_id, self.dbname)) |
|
501 self.build_db_cache(DEFAULT_EMPTY_DB_ID) |
|
502 self.restore_database(DEFAULT_EMPTY_DB_ID) |
|
503 repo = self.get_repo(startup=True) |
|
504 cnx = self.get_cnx() |
|
505 with cnx: |
|
506 pre_setup_func(cnx, self.config) |
|
507 cnx.commit() |
|
508 self.backup_database(test_db_id) |
|
509 |
|
510 |
|
511 class NoCreateDropDatabaseHandler(TestDataBaseHandler): |
|
512 """This handler is used if config.skip_db_create_and_restore is True |
|
513 |
|
514 This is typically the case with RealDBConfig. In that case, |
|
515 we explicitely want to skip init / backup / restore phases. |
|
516 |
|
517 This handler redefines the three corresponding methods and delegates |
|
518 to original handler for any other method / attribute |
|
519 """ |
|
520 |
|
521 def __init__(self, base_handler): |
|
522 self.base_handler = base_handler |
|
523 |
|
524 # override init / backup / restore methods |
|
525 def init_test_database(self): |
|
526 pass |
|
527 |
|
528 def backup_database(self, db_id): |
|
529 pass |
|
530 |
|
531 def restore_database(self, db_id): |
|
532 pass |
|
533 |
|
534 # delegate to original handler in all other cases |
|
535 def __getattr__(self, attrname): |
|
536 return getattr(self.base_handler, attrname) |
|
537 |
|
538 |
|
539 ### postgres test database handling ############################################ |
|
540 |
|
541 def startpgcluster(pyfile): |
|
542 """Start a postgresql cluster next to pyfile""" |
|
543 datadir = join(os.path.dirname(pyfile), 'data', 'database', |
|
544 'pgdb-%s' % os.path.splitext(os.path.basename(pyfile))[0]) |
|
545 if not exists(datadir): |
|
546 try: |
|
547 subprocess.check_call(['initdb', '-D', datadir, '-E', 'utf-8', '--locale=C']) |
|
548 |
|
549 except OSError as err: |
|
550 if err.errno == errno.ENOENT: |
|
551 raise OSError('"initdb" could not be found. ' |
|
552 'You should add the postgresql bin folder to your PATH ' |
|
553 '(/usr/lib/postgresql/9.1/bin for example).') |
|
554 raise |
|
555 datadir = os.path.abspath(datadir) |
|
556 pgport = '5432' |
|
557 env = os.environ.copy() |
|
558 sockdir = tempfile.mkdtemp(prefix='cwpg') |
|
559 DEFAULT_PSQL_SOURCES['system']['db-host'] = sockdir |
|
560 DEFAULT_PSQL_SOURCES['system']['db-port'] = pgport |
|
561 options = '-h "" -k %s -p %s' % (sockdir, pgport) |
|
562 options += ' -c fsync=off -c full_page_writes=off' |
|
563 options += ' -c synchronous_commit=off' |
|
564 try: |
|
565 subprocess.check_call(['pg_ctl', 'start', '-w', '-D', datadir, |
|
566 '-o', options], |
|
567 env=env) |
|
568 except OSError as err: |
|
569 try: |
|
570 os.rmdir(sockdir) |
|
571 except OSError: |
|
572 pass |
|
573 if err.errno == errno.ENOENT: |
|
574 raise OSError('"pg_ctl" could not be found. ' |
|
575 'You should add the postgresql bin folder to your PATH ' |
|
576 '(/usr/lib/postgresql/9.1/bin for example).') |
|
577 raise |
|
578 |
|
579 |
|
580 def stoppgcluster(pyfile): |
|
581 """Kill the postgresql cluster running next to pyfile""" |
|
582 datadir = join(os.path.dirname(pyfile), 'data', 'database', |
|
583 'pgdb-%s' % os.path.splitext(os.path.basename(pyfile))[0]) |
|
584 subprocess.call(['pg_ctl', 'stop', '-D', datadir, '-m', 'fast']) |
|
585 try: |
|
586 os.rmdir(DEFAULT_PSQL_SOURCES['system']['db-host']) |
|
587 except OSError: |
|
588 pass |
|
589 |
|
590 |
|
591 class PostgresTestDataBaseHandler(TestDataBaseHandler): |
|
592 DRIVER = 'postgres' |
|
593 |
|
594 # Separate db_cache for PG databases, to avoid collisions with sqlite dbs |
|
595 db_cache = {} |
|
596 explored_glob = set() |
|
597 |
|
598 __CTL = set() |
|
599 |
|
600 def __init__(self, *args, **kwargs): |
|
601 super(PostgresTestDataBaseHandler, self).__init__(*args, **kwargs) |
|
602 if 'global-db-name' not in self.system_source: |
|
603 self.system_source['global-db-name'] = self.system_source['db-name'] |
|
604 self.system_source['db-name'] = self.system_source['db-name'] + str(os.getpid()) |
|
605 |
|
606 @property |
|
607 @cached |
|
608 def helper(self): |
|
609 from logilab.database import get_db_helper |
|
610 return get_db_helper('postgres') |
|
611 |
|
612 @property |
|
613 def dbname(self): |
|
614 return self.system_source['global-db-name'] |
|
615 |
|
616 @property |
|
617 def dbcnx(self): |
|
618 try: |
|
619 return self._cnx |
|
620 except AttributeError: |
|
621 from cubicweb.server.serverctl import _db_sys_cnx |
|
622 try: |
|
623 self._cnx = _db_sys_cnx( |
|
624 self.system_source, 'CREATE DATABASE and / or USER', |
|
625 interactive=False) |
|
626 return self._cnx |
|
627 except Exception: |
|
628 self._cnx = None |
|
629 raise |
|
630 |
|
631 @property |
|
632 @cached |
|
633 def cursor(self): |
|
634 return self.dbcnx.cursor() |
|
635 |
|
636 def process_cache_entry(self, directory, dbname, db_id, entry): |
|
637 backup_name = self._backup_name(db_id) |
|
638 if backup_name in self.helper.list_databases(self.cursor): |
|
639 return backup_name |
|
640 return None |
|
641 |
|
642 def has_cache(self, db_id): |
|
643 backup_name = self._backup_name(db_id) |
|
644 return (super(PostgresTestDataBaseHandler, self).has_cache(db_id) |
|
645 and backup_name in self.helper.list_databases(self.cursor)) |
|
646 |
|
647 def init_test_database(self): |
|
648 """initialize a fresh postgresql database used for testing purpose""" |
|
649 from cubicweb.server import init_repository |
|
650 from cubicweb.server.serverctl import system_source_cnx, createdb |
|
651 # connect on the dbms system base to create our base |
|
652 try: |
|
653 self._drop(self.system_source['db-name']) |
|
654 createdb(self.helper, self.system_source, self.dbcnx, self.cursor) |
|
655 self.dbcnx.commit() |
|
656 cnx = system_source_cnx(self.system_source, special_privs='LANGUAGE C', |
|
657 interactive=False) |
|
658 templcursor = cnx.cursor() |
|
659 try: |
|
660 # XXX factorize with db-create code |
|
661 self.helper.init_fti_extensions(templcursor) |
|
662 # install plpythonu/plpgsql language if not installed by the cube |
|
663 langs = sys.platform == 'win32' and ('plpgsql',) or ('plpythonu', 'plpgsql') |
|
664 for extlang in langs: |
|
665 self.helper.create_language(templcursor, extlang) |
|
666 cnx.commit() |
|
667 finally: |
|
668 templcursor.close() |
|
669 cnx.close() |
|
670 init_repository(self.config, interactive=False, |
|
671 init_config=self.init_config) |
|
672 except BaseException: |
|
673 if self.dbcnx is not None: |
|
674 self.dbcnx.rollback() |
|
675 sys.stderr.write('building %s failed\n' % self.dbname) |
|
676 #self._drop(self.dbname) |
|
677 raise |
|
678 |
|
679 def helper_clear_cache(self): |
|
680 if self.dbcnx is not None: |
|
681 self.dbcnx.commit() |
|
682 self.dbcnx.close() |
|
683 del self._cnx |
|
684 clear_cache(self, 'cursor') |
|
685 clear_cache(self, 'helper') |
|
686 |
|
687 def __del__(self): |
|
688 self.helper_clear_cache() |
|
689 |
|
690 @property |
|
691 def _config_id(self): |
|
692 return sha1(self.config.apphome.encode('utf-8')).hexdigest()[:10] |
|
693 |
|
694 def _backup_name(self, db_id): # merge me with parent |
|
695 backup_name = '_'.join(('cache', self._config_id, self.dbname, db_id)) |
|
696 return backup_name.lower() |
|
697 |
|
698 def _drop(self, db_name): |
|
699 if db_name in self.helper.list_databases(self.cursor): |
|
700 self.cursor.execute('DROP DATABASE %s' % db_name) |
|
701 self.dbcnx.commit() |
|
702 |
|
703 def _backup_database(self, db_id): |
|
704 """Actual backup the current database. |
|
705 |
|
706 return a value to be stored in db_cache to allow restoration |
|
707 """ |
|
708 from cubicweb.server.serverctl import createdb |
|
709 orig_name = self.system_source['db-name'] |
|
710 try: |
|
711 backup_name = self._backup_name(db_id) |
|
712 self._drop(backup_name) |
|
713 self.system_source['db-name'] = backup_name |
|
714 if self._repo: |
|
715 self._repo.turn_repo_off() |
|
716 try: |
|
717 createdb(self.helper, self.system_source, self.dbcnx, self.cursor, template=orig_name) |
|
718 self.dbcnx.commit() |
|
719 finally: |
|
720 if self._repo: |
|
721 self._repo.turn_repo_on() |
|
722 return backup_name |
|
723 finally: |
|
724 self.system_source['db-name'] = orig_name |
|
725 |
|
726 def _restore_database(self, backup_coordinates, config): |
|
727 from cubicweb.server.serverctl import createdb |
|
728 """Actual restore of the current database. |
|
729 |
|
730 Use the value tostored in db_cache as input """ |
|
731 self._drop(self.system_source['db-name']) |
|
732 createdb(self.helper, self.system_source, self.dbcnx, self.cursor, |
|
733 template=backup_coordinates) |
|
734 self.dbcnx.commit() |
|
735 |
|
736 |
|
737 |
|
738 ### sqlserver2005 test database handling ####################################### |
|
739 |
|
740 class SQLServerTestDataBaseHandler(TestDataBaseHandler): |
|
741 DRIVER = 'sqlserver' |
|
742 |
|
743 # XXX complete me |
|
744 |
|
745 def init_test_database(self): |
|
746 """initialize a fresh sqlserver databse used for testing purpose""" |
|
747 if self.config.init_repository: |
|
748 from cubicweb.server import init_repository |
|
749 init_repository(self.config, interactive=False, drop=True, |
|
750 init_config=self.init_config) |
|
751 |
|
752 ### sqlite test database handling ############################################## |
|
753 |
|
754 class SQLiteTestDataBaseHandler(TestDataBaseHandler): |
|
755 DRIVER = 'sqlite' |
|
756 |
|
757 __TMPDB = set() |
|
758 |
|
759 @classmethod |
|
760 def _cleanup_all_tmpdb(cls): |
|
761 for dbpath in cls.__TMPDB: |
|
762 cls._cleanup_database(dbpath) |
|
763 |
|
764 |
|
765 |
|
766 def __init__(self, *args, **kwargs): |
|
767 super(SQLiteTestDataBaseHandler, self).__init__(*args, **kwargs) |
|
768 # use a dedicated base for each process. |
|
769 if 'global-db-name' not in self.system_source: |
|
770 self.system_source['global-db-name'] = self.system_source['db-name'] |
|
771 process_db = self.system_source['db-name'] + str(os.getpid()) |
|
772 self.system_source['db-name'] = process_db |
|
773 process_db = self.absolute_dbfile() # update db-name to absolute path |
|
774 self.__TMPDB.add(process_db) |
|
775 |
|
776 @staticmethod |
|
777 def _cleanup_database(dbfile): |
|
778 try: |
|
779 os.remove(dbfile) |
|
780 os.remove('%s-journal' % dbfile) |
|
781 except OSError: |
|
782 pass |
|
783 |
|
784 @property |
|
785 def dbname(self): |
|
786 return self.system_source['global-db-name'] |
|
787 |
|
788 def absolute_dbfile(self): |
|
789 """absolute path of current database file""" |
|
790 dbfile = join(self._ensure_test_backup_db_dir(), |
|
791 self.system_source['db-name']) |
|
792 self.system_source['db-name'] = dbfile |
|
793 return dbfile |
|
794 |
|
795 def process_cache_entry(self, directory, dbname, db_id, entry): |
|
796 return entry.get('sqlite') |
|
797 |
|
798 def _backup_database(self, db_id=DEFAULT_EMPTY_DB_ID): |
|
799 # XXX remove database file if it exists ??? |
|
800 dbfile = self.absolute_dbfile() |
|
801 backup_file = self.absolute_backup_file(db_id, 'sqlite') |
|
802 shutil.copy(dbfile, backup_file) |
|
803 # Useful to debug WHO writes a database |
|
804 # backup_stack = self.absolute_backup_file(db_id, '.stack') |
|
805 #with open(backup_stack, 'w') as backup_stack_file: |
|
806 # import traceback |
|
807 # traceback.print_stack(file=backup_stack_file) |
|
808 return backup_file |
|
809 |
|
810 def _restore_database(self, backup_coordinates, _config): |
|
811 # remove database file if it exists ? |
|
812 dbfile = self.absolute_dbfile() |
|
813 self._cleanup_database(dbfile) |
|
814 shutil.copy(backup_coordinates, dbfile) |
|
815 self.get_repo() |
|
816 |
|
817 def init_test_database(self): |
|
818 """initialize a fresh sqlite databse used for testing purpose""" |
|
819 # initialize the database |
|
820 from cubicweb.server import init_repository |
|
821 self._cleanup_database(self.absolute_dbfile()) |
|
822 init_repository(self.config, interactive=False, |
|
823 init_config=self.init_config) |
|
824 |
|
825 import atexit |
|
826 atexit.register(SQLiteTestDataBaseHandler._cleanup_all_tmpdb) |
|
827 |
|
828 |
|
829 HANDLERS = {} |
|
830 |
|
831 def register_handler(handlerkls, overwrite=False): |
|
832 assert handlerkls is not None |
|
833 if overwrite or handlerkls.DRIVER not in HANDLERS: |
|
834 HANDLERS[handlerkls.DRIVER] = handlerkls |
|
835 else: |
|
836 msg = "%s: Handler already exists use overwrite if it's intended\n"\ |
|
837 "(existing handler class is %r)" |
|
838 raise ValueError(msg % (handlerkls.DRIVER, HANDLERS[handlerkls.DRIVER])) |
|
839 |
|
840 register_handler(PostgresTestDataBaseHandler) |
|
841 register_handler(SQLiteTestDataBaseHandler) |
|
842 register_handler(SQLServerTestDataBaseHandler) |
|
843 |
|
844 |
|
845 class HCache(object): |
|
846 """Handler cache object: store database handler for a given configuration. |
|
847 |
|
848 We only keep one repo in cache to prevent too much objects to stay alive |
|
849 (database handler holds a reference to a repository). As at the moment a new |
|
850 handler is created for each TestCase class and all test methods are executed |
|
851 sequentially whithin this class, there should not have more cache miss that |
|
852 if we had a wider cache as once a Handler stop being used it won't be used |
|
853 again. |
|
854 """ |
|
855 |
|
856 def __init__(self): |
|
857 self.config = None |
|
858 self.handler = None |
|
859 |
|
860 def get(self, config): |
|
861 if config is self.config: |
|
862 return self.handler |
|
863 else: |
|
864 return None |
|
865 |
|
866 def set(self, config, handler): |
|
867 self.config = config |
|
868 self.handler = handler |
|
869 |
|
870 HCACHE = HCache() |
|
871 |
|
872 |
|
873 # XXX a class method on Test ? |
|
874 |
|
875 _CONFIG = None |
|
876 def get_test_db_handler(config, init_config=None): |
|
877 global _CONFIG |
|
878 if _CONFIG is not None and config is not _CONFIG: |
|
879 from logilab.common.modutils import cleanup_sys_modules |
|
880 # cleanup all dynamically loaded modules and everything in the instance |
|
881 # directory |
|
882 apphome = _CONFIG.apphome |
|
883 if apphome: # may be unset in tests |
|
884 cleanup_sys_modules([apphome]) |
|
885 # also cleanup sys.path |
|
886 if apphome in sys.path: |
|
887 sys.path.remove(apphome) |
|
888 _CONFIG = config |
|
889 config.adjust_sys_path() |
|
890 handler = HCACHE.get(config) |
|
891 if handler is not None: |
|
892 return handler |
|
893 driver = config.system_source_config['db-driver'] |
|
894 key = (driver, config) |
|
895 handlerkls = HANDLERS.get(driver, None) |
|
896 if handlerkls is not None: |
|
897 handler = handlerkls(config, init_config) |
|
898 if config.skip_db_create_and_restore: |
|
899 handler = NoCreateDropDatabaseHandler(handler) |
|
900 HCACHE.set(config, handler) |
|
901 return handler |
|
902 else: |
|
903 raise ValueError('no initialization function for driver %r' % driver) |
|
904 |
|
905 ### compatibility layer ############################################## |
|
906 from logilab.common.deprecation import deprecated |
|
907 |
|
908 @deprecated("please use the new DatabaseHandler mecanism") |
|
909 def init_test_database(config=None, configdir='data', apphome=None): |
|
910 """init a test database for a specific driver""" |
|
911 if config is None: |
|
912 config = TestServerConfiguration(apphome=apphome) |
|
913 handler = get_test_db_handler(config) |
|
914 handler.build_db_cache() |
|
915 return handler.get_repo_and_cnx() |
|