212 def test_something(self): |
259 def test_something(self): |
213 rset = self.execute('Any X WHERE X is CWUser') |
260 rset = self.execute('Any X WHERE X is CWUser') |
214 self.view('foaf', rset) |
261 self.view('foaf', rset) |
215 |
262 |
216 """ |
263 """ |
217 db_require_setup = False # skip init_db / reset_db steps |
|
218 read_instance_schema = True # read schema from database |
264 read_instance_schema = True # read schema from database |
219 |
265 |
220 |
266 |
221 # test database handling ####################################################### |
267 # test database handling ####################################################### |
222 |
268 |
223 def init_test_database(config=None, appid='data', apphome=None): |
269 DEFAULT_EMPTY_DB_ID = '__default_empty_db__' |
224 """init a test database for a specific driver""" |
270 |
225 from cubicweb.dbapi import in_memory_repo_cnx |
271 class TestDataBaseHandler(object): |
226 config = config or TestServerConfiguration(appid, apphome=apphome) |
272 DRIVER = None |
227 sources = config.sources() |
273 db_cache = {} |
228 driver = sources['system']['db-driver'] |
274 explored_glob = set() |
229 if config.db_require_setup: |
275 |
230 if driver == 'sqlite': |
276 def __init__(self, config): |
231 init_test_database_sqlite(config) |
277 self.config = config |
232 elif driver == 'postgres': |
278 self._repo = None |
233 init_test_database_postgres(config) |
279 # pure consistency check |
|
280 assert self.system_source['db-driver'] == self.DRIVER |
|
281 |
|
282 def _ensure_test_backup_db_dir(self): |
|
283 """Return path of directory for database backup. |
|
284 |
|
285 The function create it if necessary""" |
|
286 backupdir = join(self.config.apphome, 'database') |
|
287 if not isdir(backupdir): |
|
288 os.makedirs(backupdir) |
|
289 return backupdir |
|
290 |
|
291 def config_path(self, db_id): |
|
292 """Path for config backup of a given database id""" |
|
293 return self.absolute_backup_file(db_id, 'config') |
|
294 |
|
295 def absolute_backup_file(self, db_id, suffix): |
|
296 """Path for config backup of a given database id""" |
|
297 dbname = self.dbname.replace('-', '_') |
|
298 assert '.' not in db_id |
|
299 filename = '%s-%s.%s' % (dbname, db_id, suffix) |
|
300 return join(self._ensure_test_backup_db_dir(), filename) |
|
301 |
|
302 def db_cache_key(self, db_id, dbname=None): |
|
303 """Build a database cache key for a db_id with the current config |
|
304 |
|
305 This key is meant to be used in the cls.db_cache mapping""" |
|
306 if dbname is None: |
|
307 dbname = self.dbname |
|
308 dbname = os.path.basename(dbname) |
|
309 dbname = dbname.replace('-', '_') |
|
310 return (self.config.apphome, dbname, db_id) |
|
311 |
|
312 def backup_database(self, db_id): |
|
313 """Store the content of the current database as <db_id> |
|
314 |
|
315 The config used are also stored.""" |
|
316 backup_data = self._backup_database(db_id) |
|
317 config_path = self.config_path(db_id) |
|
318 # XXX we dump a dict of the config |
|
319 # This is an experimental to help config dependant setup (like BFSS) to |
|
320 # be propertly restored |
|
321 with open(config_path, 'wb') as conf_file: |
|
322 conf_file.write(pickle.dumps(dict(self.config))) |
|
323 self.db_cache[self.db_cache_key(db_id)] = (backup_data, config_path) |
|
324 |
|
325 def _backup_database(self, db_id): |
|
326 """Actual backup the current database. |
|
327 |
|
328 return a value to be stored in db_cache to allow restoration""" |
|
329 raise NotImplementedError() |
|
330 |
|
331 def restore_database(self, db_id): |
|
332 """Restore a database. |
|
333 |
|
334 takes as argument value stored in db_cache by self._backup_database""" |
|
335 # XXX set a clearer error message ??? |
|
336 backup_coordinates, config_path = self.db_cache[self.db_cache_key(db_id)] |
|
337 # reload the config used to create the database. |
|
338 config = pickle.loads(open(config_path, 'rb').read()) |
|
339 # shutdown repo before changing database content |
|
340 if self._repo is not None: |
|
341 self._repo.turn_repo_off() |
|
342 self._restore_database(backup_coordinates, config) |
|
343 |
|
344 def _restore_database(self, backup_coordinates, config): |
|
345 """Actual restore of the current database. |
|
346 |
|
347 Use the value tostored in db_cache as input """ |
|
348 raise NotImplementedError() |
|
349 |
|
350 def get_repo(self, startup=False): |
|
351 """ return Repository object on the current database. |
|
352 |
|
353 (turn the current repo object "on" if there is one or recreate one) |
|
354 if startup is True, server startup server hooks will be called if needed |
|
355 """ |
|
356 if self._repo is None: |
|
357 self._repo = self._new_repo(self.config) |
|
358 repo = self._repo |
|
359 repo.turn_repo_on() |
|
360 if startup and not repo._has_started: |
|
361 repo.hm.call_hooks('server_startup', repo=repo) |
|
362 repo._has_started = True |
|
363 return repo |
|
364 |
|
365 def _new_repo(self, config): |
|
366 """Factory method to create a new Repository Instance""" |
|
367 from cubicweb.dbapi import in_memory_repo |
|
368 config._cubes = None |
|
369 repo = in_memory_repo(config) |
|
370 # extending Repository class |
|
371 repo._has_started = False |
|
372 repo._needs_refresh = False |
|
373 repo.turn_repo_on = partial(turn_repo_on, repo) |
|
374 repo.turn_repo_off = partial(turn_repo_off, repo) |
|
375 return repo |
|
376 |
|
377 |
|
378 def get_cnx(self): |
|
379 """return Connection object ont he current repository""" |
|
380 from cubicweb.dbapi import in_memory_cnx |
|
381 repo = self.get_repo() |
|
382 sources = self.config.sources() |
|
383 login = unicode(sources['admin']['login']) |
|
384 password = sources['admin']['password'] or 'xxx' |
|
385 cnx = in_memory_cnx(repo, login, password=password) |
|
386 return cnx |
|
387 |
|
388 def get_repo_and_cnx(self, db_id=DEFAULT_EMPTY_DB_ID): |
|
389 """Reset database with the current db_id and return (repo, cnx) |
|
390 |
|
391 A database *MUST* have been build with the current <db_id> prior to |
|
392 call this method. See the ``build_db_cache`` method. The returned |
|
393 repository have it's startup hooks called and the connection is |
|
394 establised as admin.""" |
|
395 |
|
396 self.restore_database(db_id) |
|
397 repo = self.get_repo(startup=True) |
|
398 cnx = self.get_cnx() |
|
399 return repo, cnx |
|
400 |
|
401 @property |
|
402 def system_source(self): |
|
403 sources = self.config.sources() |
|
404 return sources['system'] |
|
405 |
|
406 @property |
|
407 def dbname(self): |
|
408 return self.system_source['db-name'] |
|
409 |
|
410 def init_test_database(): |
|
411 """actual initialisation of the database""" |
|
412 raise ValueError('no initialization function for driver %r' % driver) |
|
413 |
|
414 def has_cache(self, db_id): |
|
415 """Check if a given database id exist in cb cache for the current config""" |
|
416 cache_glob = self.absolute_backup_file('*', '*') |
|
417 if cache_glob not in self.explored_glob: |
|
418 self.discover_cached_db() |
|
419 return self.db_cache_key(db_id) in self.db_cache |
|
420 |
|
421 def discover_cached_db(self): |
|
422 """Search available db_if for the current config""" |
|
423 cache_glob = self.absolute_backup_file('*', '*') |
|
424 directory = os.path.dirname(cache_glob) |
|
425 entries={} |
|
426 candidates = glob.glob(cache_glob) |
|
427 for filepath in candidates: |
|
428 data = os.path.basename(filepath) |
|
429 # database backup are in the forms are <dbname>-<db_id>.<backtype> |
|
430 dbname, data = data.split('-', 1) |
|
431 db_id, filetype = data.split('.', 1) |
|
432 entries.setdefault((dbname, db_id), {})[filetype] = filepath |
|
433 for (dbname, db_id), entry in entries.iteritems(): |
|
434 # apply necessary transformation from the driver |
|
435 value = self.process_cache_entry(directory, dbname, db_id, entry) |
|
436 assert 'config' in entry |
|
437 if value is not None: # None value means "not handled by this driver |
|
438 # XXX Ignored value are shadowed to other Handler if cache are common. |
|
439 key = self.db_cache_key(db_id, dbname=dbname) |
|
440 self.db_cache[key] = value, entry['config'] |
|
441 self.explored_glob.add(cache_glob) |
|
442 |
|
443 def process_cache_entry(self, directory, dbname, db_id, entry): |
|
444 """Transforms potential cache entry to proper backup coordinate |
|
445 |
|
446 entry argument is a "filetype" -> "filepath" mapping |
|
447 Return None if an entry should be ignored.""" |
|
448 return None |
|
449 |
|
450 def build_db_cache(self, test_db_id=DEFAULT_EMPTY_DB_ID, pre_setup_func=None): |
|
451 """Build Database cache for ``test_db_id`` if a cache doesn't exist |
|
452 |
|
453 if ``test_db_id is DEFAULT_EMPTY_DB_ID`` self.init_test_database is |
|
454 called. otherwise, DEFAULT_EMPTY_DB_ID is build/restored and |
|
455 ``pre_setup_func`` to setup the database. |
|
456 |
|
457 This function backup any database it build""" |
|
458 |
|
459 if self.has_cache(test_db_id): |
|
460 return #test_db_id, 'already in cache' |
|
461 if test_db_id is DEFAULT_EMPTY_DB_ID: |
|
462 self.init_test_database() |
234 else: |
463 else: |
235 raise ValueError('no initialization function for driver %r' % driver) |
464 print 'Building %s for database %s' % (test_db_id, self.dbname) |
236 config._cubes = None # avoid assertion error |
465 self.build_db_cache(DEFAULT_EMPTY_DB_ID) |
237 repo, cnx = in_memory_repo_cnx(config, unicode(sources['admin']['login']), |
466 self.restore_database(DEFAULT_EMPTY_DB_ID) |
238 password=sources['admin']['password'] or 'xxx') |
467 repo = self.get_repo(startup=True) |
239 if driver == 'sqlite': |
468 cnx = self.get_cnx() |
240 install_sqlite_patch(repo.querier) |
469 session = repo._sessions[cnx.sessionid] |
241 return repo, cnx |
470 session.set_pool() |
242 |
471 _commit = session.commit |
243 def reset_test_database(config): |
472 def always_pooled_commit(): |
244 """init a test database for a specific driver""" |
473 _commit() |
245 if not config.db_require_setup: |
474 session.set_pool() |
246 return |
475 session.commit = always_pooled_commit |
247 driver = config.sources()['system']['db-driver'] |
476 pre_setup_func(session, self.config) |
248 if driver == 'sqlite': |
477 session.commit() |
249 reset_test_database_sqlite(config) |
478 cnx.close() |
250 elif driver == 'postgres': |
479 self.backup_database(test_db_id) |
251 init_test_database_postgres(config) |
|
252 else: |
|
253 raise ValueError('no reset function for driver %r' % driver) |
|
254 |
|
255 |
480 |
256 ### postgres test database handling ############################################ |
481 ### postgres test database handling ############################################ |
257 |
482 |
258 def init_test_database_postgres(config): |
483 class PostgresTestDataBaseHandler(TestDataBaseHandler): |
259 """initialize a fresh postgresql databse used for testing purpose""" |
484 |
260 from logilab.database import get_db_helper |
485 # XXX |
261 from cubicweb.server import init_repository |
486 # XXX PostgresTestDataBaseHandler Have not been tested at all. |
262 from cubicweb.server.serverctl import (createdb, system_source_cnx, |
487 # XXX |
263 _db_sys_cnx) |
488 DRIVER = 'postgres' |
264 source = config.sources()['system'] |
489 |
265 dbname = source['db-name'] |
490 @property |
266 templdbname = dbname + '_template' |
491 @cached |
267 helper = get_db_helper('postgres') |
492 def helper(self): |
268 # connect on the dbms system base to create our base |
493 from logilab.database import get_db_helper |
269 dbcnx = _db_sys_cnx(source, 'CREATE DATABASE and / or USER', verbose=0) |
494 return get_db_helper('postgres') |
270 cursor = dbcnx.cursor() |
495 |
271 try: |
496 @property |
272 if dbname in helper.list_databases(cursor): |
497 @cached |
273 cursor.execute('DROP DATABASE %s' % dbname) |
498 def dbcnx(self): |
274 if not templdbname in helper.list_databases(cursor): |
499 from cubicweb.server.serverctl import _db_sys_cnx |
275 source['db-name'] = templdbname |
500 return _db_sys_cnx(self.system_source, 'CREATE DATABASE and / or USER', verbose=0) |
276 createdb(helper, source, dbcnx, cursor) |
501 |
277 dbcnx.commit() |
502 @property |
278 cnx = system_source_cnx(source, special_privs='LANGUAGE C', verbose=0) |
503 @cached |
|
504 def cursor(self): |
|
505 return self.dbcnx.cursor() |
|
506 |
|
507 def init_test_database(self): |
|
508 """initialize a fresh postgresql databse used for testing purpose""" |
|
509 from cubicweb.server import init_repository |
|
510 from cubicweb.server.serverctl import system_source_cnx, createdb |
|
511 # connect on the dbms system base to create our base |
|
512 try: |
|
513 self._drop(self.dbname) |
|
514 |
|
515 createdb(self.helper, self.system_source, self.dbcnx, self.cursor) |
|
516 self.dbcnx.commit() |
|
517 cnx = system_source_cnx(self.system_source, special_privs='LANGUAGE C', verbose=0) |
279 templcursor = cnx.cursor() |
518 templcursor = cnx.cursor() |
280 # XXX factorize with db-create code |
519 try: |
281 helper.init_fti_extensions(templcursor) |
520 # XXX factorize with db-create code |
282 # install plpythonu/plpgsql language if not installed by the cube |
521 self.helper.init_fti_extensions(templcursor) |
283 langs = sys.platform == 'win32' and ('plpgsql',) or ('plpythonu', 'plpgsql') |
522 # install plpythonu/plpgsql language if not installed by the cube |
284 for extlang in langs: |
523 langs = sys.platform == 'win32' and ('plpgsql',) or ('plpythonu', 'plpgsql') |
285 helper.create_language(templcursor, extlang) |
524 for extlang in langs: |
286 cnx.commit() |
525 self.helper.create_language(templcursor, extlang) |
287 templcursor.close() |
526 cnx.commit() |
288 cnx.close() |
527 finally: |
289 init_repository(config, interactive=False) |
528 templcursor.close() |
290 source['db-name'] = dbname |
529 cnx.close() |
291 except: |
530 init_repository(self.config, interactive=False) |
292 dbcnx.rollback() |
531 except: |
293 # XXX drop template |
532 self.dbcnx.rollback() |
294 raise |
533 print >> sys.stderr, 'building', self.dbname, 'failed' |
295 createdb(helper, source, dbcnx, cursor, template=templdbname) |
534 #self._drop(self.dbname) |
296 dbcnx.commit() |
535 raise |
297 dbcnx.close() |
536 |
|
537 def helper_clear_cache(self): |
|
538 self.dbcnx.commit() |
|
539 self.dbcnx.close() |
|
540 clear_cache(self, 'dbcnx') |
|
541 clear_cache(self, 'helper') |
|
542 clear_cache(self, 'cursor') |
|
543 |
|
544 def __del__(self): |
|
545 self.helper_clear_cache() |
|
546 |
|
547 @property |
|
548 def _config_id(self): |
|
549 return hashlib.sha1(self.config.apphome).hexdigest()[:10] |
|
550 |
|
551 def _backup_name(self, db_id): # merge me with parent |
|
552 backup_name = '_'.join(('cache', self._config_id, self.dbname, db_id)) |
|
553 return backup_name.lower() |
|
554 |
|
555 def _drop(self, db_name): |
|
556 if db_name in self.helper.list_databases(self.cursor): |
|
557 #print 'dropping overwritted database:', db_name |
|
558 self.cursor.execute('DROP DATABASE %s' % db_name) |
|
559 self.dbcnx.commit() |
|
560 |
|
561 def _backup_database(self, db_id): |
|
562 """Actual backup the current database. |
|
563 |
|
564 return a value to be stored in db_cache to allow restoration""" |
|
565 from cubicweb.server.serverctl import createdb |
|
566 orig_name = self.system_source['db-name'] |
|
567 try: |
|
568 backup_name = self._backup_name(db_id) |
|
569 #print 'storing postgres backup as', backup_name |
|
570 self._drop(backup_name) |
|
571 self.system_source['db-name'] = backup_name |
|
572 createdb(self.helper, self.system_source, self.dbcnx, self.cursor, template=orig_name) |
|
573 self.dbcnx.commit() |
|
574 return backup_name |
|
575 finally: |
|
576 self.system_source['db-name'] = orig_name |
|
577 |
|
578 def _restore_database(self, backup_coordinates, config): |
|
579 from cubicweb.server.serverctl import createdb |
|
580 """Actual restore of the current database. |
|
581 |
|
582 Use the value tostored in db_cache as input """ |
|
583 #print 'restoring postgrest backup from', backup_coordinates |
|
584 self._drop(self.dbname) |
|
585 createdb(self.helper, self.system_source, self.dbcnx, self.cursor, |
|
586 template=backup_coordinates) |
|
587 self.dbcnx.commit() |
|
588 |
|
589 |
298 |
590 |
299 ### sqlserver2005 test database handling ####################################### |
591 ### sqlserver2005 test database handling ####################################### |
300 |
592 |
301 def init_test_database_sqlserver2005(config): |
593 class SQLServerTestDataBaseHandler(TestDataBaseHandler): |
302 """initialize a fresh sqlserver databse used for testing purpose""" |
594 DRIVER = 'sqlserver' |
303 if config.init_repository: |
595 |
304 from cubicweb.server import init_repository |
596 # XXX complete me |
305 init_repository(config, interactive=False, drop=True) |
597 |
|
598 def init_test_database(self): |
|
599 """initialize a fresh sqlserver databse used for testing purpose""" |
|
600 if self.config.init_repository: |
|
601 from cubicweb.server import init_repository |
|
602 init_repository(config, interactive=False, drop=True) |
306 |
603 |
307 ### sqlite test database handling ############################################## |
604 ### sqlite test database handling ############################################## |
308 |
605 |
309 def cleanup_sqlite(dbfile, removetemplate=False): |
606 class SQLiteTestDataBaseHandler(TestDataBaseHandler): |
310 try: |
607 DRIVER = 'sqlite' |
311 os.remove(dbfile) |
608 |
312 os.remove('%s-journal' % dbfile) |
609 @staticmethod |
313 except OSError: |
610 def _cleanup_database(dbfile): |
314 pass |
|
315 if removetemplate: |
|
316 try: |
611 try: |
317 os.remove('%s-template' % dbfile) |
612 os.remove(dbfile) |
|
613 os.remove('%s-journal' % dbfile) |
318 except OSError: |
614 except OSError: |
319 pass |
615 pass |
320 |
616 |
321 def reset_test_database_sqlite(config): |
617 def absolute_dbfile(self): |
322 import shutil |
618 """absolute path of current database file""" |
323 dbfile = config.sources()['system']['db-name'] |
619 dbfile = join(self._ensure_test_backup_db_dir(), |
324 cleanup_sqlite(dbfile) |
620 self.config.sources()['system']['db-name']) |
325 template = '%s-template' % dbfile |
621 self.config.sources()['system']['db-name'] = dbfile |
326 if exists(template): |
622 return dbfile |
327 shutil.copy(template, dbfile) |
623 |
328 return True |
624 |
329 return False |
625 def process_cache_entry(self, directory, dbname, db_id, entry): |
330 |
626 return entry.get('sqlite') |
331 def init_test_database_sqlite(config): |
627 |
332 """initialize a fresh sqlite databse used for testing purpose""" |
628 def _backup_database(self, db_id=DEFAULT_EMPTY_DB_ID): |
333 # remove database file if it exists |
629 # XXX remove database file if it exists ??? |
334 dbfile = join(config.apphome, config.sources()['system']['db-name']) |
630 dbfile = self.absolute_dbfile() |
335 config.sources()['system']['db-name'] = dbfile |
631 backup_file = self.absolute_backup_file(db_id, 'sqlite') |
336 if not reset_test_database_sqlite(config): |
632 shutil.copy(dbfile, backup_file) |
|
633 # Usefull to debug WHO write a database |
|
634 # backup_stack = self.absolute_backup_file(db_id, '.stack') |
|
635 #with open(backup_stack, 'w') as backup_stack_file: |
|
636 # import traceback |
|
637 # traceback.print_stack(file=backup_stack_file) |
|
638 return backup_file |
|
639 |
|
640 def _new_repo(self, config): |
|
641 repo = super(SQLiteTestDataBaseHandler, self)._new_repo(config) |
|
642 install_sqlite_patch(repo.querier) |
|
643 return repo |
|
644 |
|
645 def _restore_database(self, backup_coordinates, _config): |
|
646 # remove database file if it exists ? |
|
647 dbfile = self.absolute_dbfile() |
|
648 self._cleanup_database(dbfile) |
|
649 #print 'resto from', backup_coordinates |
|
650 shutil.copy(backup_coordinates, dbfile) |
|
651 repo = self.get_repo() |
|
652 |
|
653 def init_test_database(self): |
|
654 """initialize a fresh sqlite databse used for testing purpose""" |
337 # initialize the database |
655 # initialize the database |
338 import shutil |
|
339 from cubicweb.server import init_repository |
656 from cubicweb.server import init_repository |
340 init_repository(config, interactive=False) |
657 self._cleanup_database(self.absolute_dbfile()) |
341 shutil.copy(dbfile, '%s-template' % dbfile) |
658 init_repository(self.config, interactive=False) |
|
659 |
342 |
660 |
343 def install_sqlite_patch(querier): |
661 def install_sqlite_patch(querier): |
344 """This patch hotfixes the following sqlite bug : |
662 """This patch hotfixes the following sqlite bug : |
345 - http://www.sqlite.org/cvstrac/tktview?tn=1327,33 |
663 - http://www.sqlite.org/cvstrac/tktview?tn=1327,33 |
346 (some dates are returned as strings rather thant date objects) |
664 (some dates are returned as strings rather thant date objects) |