Make test database template creation concurrent
build_db_cache() is used in tests to create test database templates, i.e.
DEFAULT_EMPTY_DB_ID (which is __default_empty_db__) and custom template
database using CubicwebTC test_db_id/pre_setup_database API.
When running tests in parallel using multiple processes, build_db_cache() may
try to build the same database twice. Avoid this by adding synchronisation of
process by using a file lock.
So when two processes require the same template database, one build the
database and others wait it to be created.
Use filelock (https://github.com/benediktschmitt/py-filelock) library to have a
portable (unix / windows) way for handling locks.
Also filelock is packaged in debian: https://packages.debian.org/source/python-filelock
--- a/cubicweb/devtools/__init__.py Fri Oct 26 17:00:05 2018 +0200
+++ b/cubicweb/devtools/__init__.py Fri Oct 26 17:12:26 2018 +0200
@@ -32,6 +32,7 @@
from os.path import abspath, join, exists, split, isdir, dirname
from functools import partial
+import filelock
from six import text_type
from six.moves import cPickle as pickle
@@ -470,20 +471,23 @@
``pre_setup_func`` to setup the database.
This function backup any database it build"""
- if self.has_cache(test_db_id):
- return # test_db_id, 'already in cache'
- if test_db_id is DEFAULT_EMPTY_DB_ID:
- self.init_test_database()
- else:
- print('Building %s for database %s' % (test_db_id, self.dbname))
- self.build_db_cache(DEFAULT_EMPTY_DB_ID)
- self.restore_database(DEFAULT_EMPTY_DB_ID)
- self.get_repo(startup=True)
- cnx = self.get_cnx()
- with cnx:
- pre_setup_func(cnx, self.config)
- cnx.commit()
- self.backup_database(test_db_id)
+ lockfile = join(self._ensure_test_backup_db_dir(),
+ '{}.lock'.format(test_db_id))
+ with filelock.FileLock(lockfile):
+ if self.has_cache(test_db_id):
+ return # test_db_id, 'already in cache'
+ if test_db_id is DEFAULT_EMPTY_DB_ID:
+ self.init_test_database()
+ else:
+ print('Building %s for database %s' % (test_db_id, self.dbname))
+ self.build_db_cache(DEFAULT_EMPTY_DB_ID)
+ self.restore_database(DEFAULT_EMPTY_DB_ID)
+ self.get_repo(startup=True)
+ cnx = self.get_cnx()
+ with cnx:
+ pre_setup_func(cnx, self.config)
+ cnx.commit()
+ self.backup_database(test_db_id)
class NoCreateDropDatabaseHandler(TestDataBaseHandler):
--- a/debian/control Fri Oct 26 17:00:05 2018 +0200
+++ b/debian/control Fri Oct 26 17:12:26 2018 +0200
@@ -28,6 +28,7 @@
python-passlib,
python-repoze.lru,
python-wsgicors,
+ python-filelock,
sphinx-common,
Standards-Version: 3.9.6
Homepage: https://www.cubicweb.org
@@ -51,6 +52,7 @@
python-passlib,
python-tz,
graphviz,
+ python-filelock,
gettext,
Recommends:
cubicweb-ctl (= ${source:Version}),
--- a/setup.py Fri Oct 26 17:00:05 2018 +0200
+++ b/setup.py Fri Oct 26 17:12:26 2018 +0200
@@ -74,6 +74,7 @@
'pytz',
'Markdown',
'unittest2 >= 0.7.0',
+ 'filelock',
],
entry_points={
'console_scripts': [