--- a/cubicweb/__pkginfo__.py Fri Mar 08 14:35:59 2019 +0100
+++ b/cubicweb/__pkginfo__.py Mon Mar 11 14:34:14 2019 +0100
@@ -22,8 +22,8 @@
modname = distname = "cubicweb"
-numversion = (3, 26, 8)
-version = '.'.join(str(num) for num in numversion)
+numversion = (3, 27, 0)
+version = '.'.join(str(num) for num in numversion) + '.dev0'
description = "a repository of entities / relations for knowledge management"
author = "Logilab"
--- a/cubicweb/devtools/__init__.py Fri Mar 08 14:35:59 2019 +0100
+++ b/cubicweb/devtools/__init__.py Mon Mar 11 14:34:14 2019 +0100
@@ -32,6 +32,7 @@
from os.path import abspath, join, exists, split, isdir, dirname
from functools import partial
+import filelock
from six import text_type
from six.moves import cPickle as pickle
@@ -427,11 +428,12 @@
raise ValueError('no initialization function for driver %r' % self.DRIVER)
def has_cache(self, db_id):
- """Check if a given database id exist in cb cache for the current config"""
- cache_glob = self.absolute_backup_file('*', '*')
- if cache_glob not in self.explored_glob:
- self.discover_cached_db()
- return self.db_cache_key(db_id) in self.db_cache
+ """Check if a given database id exist in db cache for the current config"""
+ key = self.db_cache_key(db_id)
+ if key in self.db_cache:
+ return True
+ self.discover_cached_db()
+ return key in self.db_cache
def discover_cached_db(self):
"""Search available db_if for the current config"""
@@ -469,20 +471,23 @@
``pre_setup_func`` to setup the database.
This function backup any database it build"""
- if self.has_cache(test_db_id):
- return # test_db_id, 'already in cache'
- if test_db_id is DEFAULT_EMPTY_DB_ID:
- self.init_test_database()
- else:
- print('Building %s for database %s' % (test_db_id, self.dbname))
- self.build_db_cache(DEFAULT_EMPTY_DB_ID)
- self.restore_database(DEFAULT_EMPTY_DB_ID)
- self.get_repo(startup=True)
- cnx = self.get_cnx()
- with cnx:
- pre_setup_func(cnx, self.config)
- cnx.commit()
- self.backup_database(test_db_id)
+ lockfile = join(self._ensure_test_backup_db_dir(),
+ '{}.lock'.format(test_db_id))
+ with filelock.FileLock(lockfile):
+ if self.has_cache(test_db_id):
+ return # test_db_id, 'already in cache'
+ if test_db_id is DEFAULT_EMPTY_DB_ID:
+ self.init_test_database()
+ else:
+ print('Building %s for database %s' % (test_db_id, self.dbname))
+ self.build_db_cache(DEFAULT_EMPTY_DB_ID)
+ self.restore_database(DEFAULT_EMPTY_DB_ID)
+ self.get_repo(startup=True)
+ cnx = self.get_cnx()
+ with cnx:
+ pre_setup_func(cnx, self.config)
+ cnx.commit()
+ self.backup_database(test_db_id)
class NoCreateDropDatabaseHandler(TestDataBaseHandler):
--- a/cubicweb/rset.py Fri Mar 08 14:35:59 2019 +0100
+++ b/cubicweb/rset.py Mon Mar 11 14:34:14 2019 +0100
@@ -1,4 +1,4 @@
-# copyright 2003-2016 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# copyright 2003-2018 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
@@ -393,6 +393,8 @@
if self.rows[i][col] is not None:
yield self.get_entity(i, col)
+ all = entities
+
def iter_rows_with_entities(self):
""" iterates over rows, and for each row
eids are converted to plain entities
@@ -459,6 +461,34 @@
else:
raise MultipleResultsError("Multiple rows were found for one()")
+ def first(self, col=0):
+ """Retrieve the first entity from the query.
+
+ If the result set is empty, raises :exc:`NoResultError`.
+
+ :type col: int
+ :param col: The column localising the entity in the unique row
+
+ :return: the partially initialized `Entity` instance
+ """
+ if len(self) == 0:
+ raise NoResultError("No row was found for first()")
+ return self.get_entity(0, col)
+
+ def last(self, col=0):
+ """Retrieve the last entity from the query.
+
+ If the result set is empty, raises :exc:`NoResultError`.
+
+ :type col: int
+ :param col: The column localising the entity in the unique row
+
+ :return: the partially initialized `Entity` instance
+ """
+ if len(self) == 0:
+ raise NoResultError("No row was found for last()")
+ return self.get_entity(-1, col)
+
def _make_entity(self, row, col):
"""Instantiate an entity, and store it in the entity cache"""
# build entity instance
--- a/cubicweb/statsd_logger.py Fri Mar 08 14:35:59 2019 +0100
+++ b/cubicweb/statsd_logger.py Mon Mar 11 14:34:14 2019 +0100
@@ -58,6 +58,7 @@
import time
import socket
+from contextlib import contextmanager
_bucket = 'cubicweb'
_address = None
@@ -87,19 +88,32 @@
_socket = socket.socket(family, socket.SOCK_DGRAM)
+def teardown():
+ """Unconfigure the statsd endpoint
+
+ This is most likely only useful for unit tests"""
+ global _bucket, _address, _socket
+ _bucket = 'cubicweb'
+ _address = None
+ _socket = None
+
+
def statsd_c(context, n=1):
if _address is not None:
- _socket.sendto('{0}.{1}:{2}|c\n'.format(_bucket, context, n), _address)
+ _socket.sendto('{0}.{1}:{2}|c\n'.format(_bucket, context, n).encode(),
+ _address)
def statsd_g(context, value):
if _address is not None:
- _socket.sendto('{0}.{1}:{2}|g\n'.format(_bucket, context, value), _address)
+ _socket.sendto('{0}.{1}:{2}|g\n'.format(_bucket, context, value).encode(),
+ _address)
def statsd_t(context, value):
if _address is not None:
- _socket.sendto('{0}.{1}:{2:.4f}|ms\n'.format(_bucket, context, value), _address)
+ _socket.sendto('{0}.{1}:{2:.4f}|ms\n'.format(_bucket, context, value).encode(),
+ _address)
class statsd_timeit(object):
@@ -125,7 +139,7 @@
finally:
dt = 1000 * (time.time() - t0)
msg = '{0}.{1}:{2:.4f}|ms\n{0}.{1}:1|c\n'.format(
- _bucket, self.__name__, dt)
+ _bucket, self.__name__, dt).encode()
_socket.sendto(msg, _address)
def __get__(self, obj, objtype):
@@ -134,3 +148,17 @@
return self
import functools
return functools.partial(self.__call__, obj)
+
+
+@contextmanager
+def statsd_timethis(ctxmsg):
+ if _address is not None:
+ t0 = time.time()
+ try:
+ yield
+ finally:
+ if _address is not None:
+ dt = 1000 * (time.time() - t0)
+ msg = '{0}.{1}:{2:.4f}|ms\n{0}.{1}:1|c\n'.format(
+ _bucket, ctxmsg, dt).encode()
+ _socket.sendto(msg, _address)
--- a/cubicweb/test/unittest_rset.py Fri Mar 08 14:35:59 2019 +0100
+++ b/cubicweb/test/unittest_rset.py Mon Mar 11 14:34:14 2019 +0100
@@ -1,5 +1,5 @@
# coding: utf-8
-# copyright 2003-2016 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# copyright 2003-2018 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
@@ -444,6 +444,78 @@
with self.assertRaises(MultipleResultsError):
req.execute('Any X WHERE X is CWUser').one()
+ def test_first(self):
+ with self.admin_access.web_request() as req:
+ req.create_entity('CWUser',
+ login=u'cdevienne',
+ upassword=u'cdevienne',
+ surname=u'de Vienne',
+ firstname=u'Christophe')
+ e = req.execute('Any X WHERE X login "cdevienne"').first()
+ self.assertEqual(e.surname, u'de Vienne')
+
+ e = req.execute(
+ 'Any X, N WHERE X login "cdevienne", X surname N').first()
+ self.assertEqual(e.surname, u'de Vienne')
+
+ e = req.execute(
+ 'Any N, X WHERE X login "cdevienne", X surname N').first(col=1)
+ self.assertEqual(e.surname, u'de Vienne')
+
+ def test_first_no_rows(self):
+ with self.admin_access.web_request() as req:
+ with self.assertRaises(NoResultError):
+ req.execute('Any X WHERE X login "patanok"').first()
+
+ def test_first_multiple_rows(self):
+ with self.admin_access.web_request() as req:
+ req.create_entity(
+ 'CWUser', login=u'user1', upassword=u'cdevienne',
+ surname=u'de Vienne', firstname=u'Christophe')
+ req.create_entity(
+ 'CWUser', login=u'user2', upassword='adim',
+ surname=u'di mascio', firstname=u'adrien')
+
+ e = req.execute('Any X ORDERBY X WHERE X is CWUser, '
+ 'X login LIKE "user%"').first()
+ self.assertEqual(e.login, 'user1')
+
+ def test_last(self):
+ with self.admin_access.web_request() as req:
+ req.create_entity('CWUser',
+ login=u'cdevienne',
+ upassword=u'cdevienne',
+ surname=u'de Vienne',
+ firstname=u'Christophe')
+ e = req.execute('Any X WHERE X login "cdevienne"').last()
+ self.assertEqual(e.surname, u'de Vienne')
+
+ e = req.execute(
+ 'Any X, N WHERE X login "cdevienne", X surname N').last()
+ self.assertEqual(e.surname, u'de Vienne')
+
+ e = req.execute(
+ 'Any N, X WHERE X login "cdevienne", X surname N').last(col=1)
+ self.assertEqual(e.surname, u'de Vienne')
+
+ def test_last_no_rows(self):
+ with self.admin_access.web_request() as req:
+ with self.assertRaises(NoResultError):
+ req.execute('Any X WHERE X login "patanok"').last()
+
+ def test_last_multiple_rows(self):
+ with self.admin_access.web_request() as req:
+ req.create_entity(
+ 'CWUser', login=u'user1', upassword=u'cdevienne',
+ surname=u'de Vienne', firstname=u'Christophe')
+ req.create_entity(
+ 'CWUser', login=u'user2', upassword='adim',
+ surname=u'di mascio', firstname=u'adrien')
+
+ e = req.execute('Any X ORDERBY X WHERE X is CWUser, '
+ 'X login LIKE "user%"').last()
+ self.assertEqual(e.login, 'user2')
+
def test_related_entity_optional(self):
with self.admin_access.web_request() as req:
req.create_entity('Bookmark', title=u'aaaa', path=u'path')
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/cubicweb/test/unittest_statsd.py Mon Mar 11 14:34:14 2019 +0100
@@ -0,0 +1,136 @@
+# -*- coding: utf-8 -*-
+# copyright 2018 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of CubicWeb.
+#
+# CubicWeb is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option)
+# any later version.
+#
+# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
+"""unit tests for module cubicweb.statsd_logger"""
+
+import threading
+import socket
+import time
+import re
+
+from unittest import TestCase
+from cubicweb import statsd_logger as statsd
+
+
+UDP_PORT = None
+RUNNING = True
+SOCK = socket.socket(socket.AF_INET,
+ socket.SOCK_DGRAM)
+SOCK.settimeout(0.1)
+STATSD = None
+DATA = []
+
+
+def statsd_rcv():
+ while RUNNING:
+ try:
+ data, addr = SOCK.recvfrom(1024)
+ if data:
+ rcv = [row.strip().decode() for row in data.splitlines()]
+ DATA.extend(rcv)
+ except socket.timeout:
+ pass
+
+
+def setUpModule(*args):
+ global UDP_PORT, STATSD
+ SOCK.bind(('127.0.0.1', 0))
+ UDP_PORT = SOCK.getsockname()[1]
+ STATSD = threading.Thread(target=statsd_rcv)
+ STATSD.start()
+ statsd.setup('test', ('127.0.0.1', UDP_PORT))
+
+
+def tearDownModule(*args):
+ global RUNNING
+ RUNNING = False
+ STATSD.join()
+ statsd.teardown()
+
+
+class StatsdTC(TestCase):
+
+ def setUp(self):
+ super(StatsdTC, self).setUp()
+ DATA[:] = []
+
+ def check_received(self, value):
+ for i in range(10):
+ if value in DATA:
+ break
+ time.sleep(0.01)
+ else:
+ self.assertIn(value, DATA)
+
+ def check_received_ms(self, value):
+ value = re.compile(value.replace('?', r'\d'))
+ for i in range(10):
+ if [x for x in DATA if value.match(x)]:
+ break
+ time.sleep(0.01)
+ else:
+ self.assertTrue([x for x in DATA if value.match(x)], DATA)
+
+ def test_statsd_c(self):
+ statsd.statsd_c('context')
+ self.check_received('test.context:1|c')
+ statsd.statsd_c('context', 10)
+ self.check_received('test.context:10|c')
+
+ def test_statsd_g(self):
+ statsd.statsd_g('context', 42)
+ self.check_received('test.context:42|g')
+ statsd.statsd_g('context', 'Igorrr')
+ self.check_received('test.context:Igorrr|g')
+
+ def test_statsd_t(self):
+ statsd.statsd_t('context', 1)
+ self.check_received('test.context:1.0000|ms')
+ statsd.statsd_t('context', 10)
+ self.check_received('test.context:10.0000|ms')
+ statsd.statsd_t('context', 0.12344)
+ self.check_received('test.context:0.1234|ms')
+ statsd.statsd_t('context', 0.12345)
+ self.check_received('test.context:0.1235|ms')
+
+ def test_decorator(self):
+
+ @statsd.statsd_timeit
+ def measure_me_please():
+ "some nice function"
+ return 42
+
+ self.assertEqual(measure_me_please.__doc__,
+ "some nice function")
+
+ measure_me_please()
+ self.check_received_ms('test.measure_me_please:0.0???|ms')
+ self.check_received('test.measure_me_please:1|c')
+
+ def test_context_manager(self):
+
+ with statsd.statsd_timethis('cm'):
+ time.sleep(0.1)
+
+ self.check_received_ms('test.cm:100.????|ms')
+ self.check_received('test.cm:1|c')
+
+
+if __name__ == '__main__':
+ from unittest import main
+ main()
--- a/cubicweb/test/unittest_utils.py Fri Mar 08 14:35:59 2019 +0100
+++ b/cubicweb/test/unittest_utils.py Mon Mar 11 14:34:14 2019 +0100
@@ -109,6 +109,95 @@
'itemcount': 10,
'permanentcount': 5})
+ def test_clear_on_overflow(self):
+ """Tests that only non-permanent items in the cache are wiped-out on ceiling overflow
+ """
+ c = QueryCache(ceiling=10)
+ # set 10 values
+ for x in range(10):
+ c[x] = x
+ # arrange for the first 5 to be permanent
+ for x in range(5):
+ for r in range(QueryCache._maxlevel + 2):
+ v = c[x]
+ self.assertEqual(v, x)
+ # Add the 11-th
+ c[10] = 10
+ self.assertEqual(c._usage_report(),
+ {'transientcount': 0,
+ 'itemcount': 6,
+ 'permanentcount': 5})
+
+ def test_get_with_default(self):
+ """
+ Tests the capability of QueryCache for retrieving items with a default value
+ """
+ c = QueryCache(ceiling=20)
+ # set 10 values
+ for x in range(10):
+ c[x] = x
+ # arrange for the first 5 to be permanent
+ for x in range(5):
+ for r in range(QueryCache._maxlevel + 2):
+ v = c[x]
+ self.assertEqual(v, x)
+ self.assertEqual(c._usage_report(),
+ {'transientcount': 0,
+ 'itemcount': 10,
+ 'permanentcount': 5})
+ # Test defaults for existing (including in permanents)
+ for x in range(10):
+ v = c.get(x, -1)
+ self.assertEqual(v, x)
+ # Test defaults for others
+ for x in range(10, 15):
+ v = c.get(x, -1)
+ self.assertEqual(v, -1)
+
+ def test_iterkeys(self):
+ """
+ Tests the iterating on keys in the cache
+ """
+ c = QueryCache(ceiling=20)
+ # set 10 values
+ for x in range(10):
+ c[x] = x
+ # arrange for the first 5 to be permanent
+ for x in range(5):
+ for r in range(QueryCache._maxlevel + 2):
+ v = c[x]
+ self.assertEqual(v, x)
+ self.assertEqual(c._usage_report(),
+ {'transientcount': 0,
+ 'itemcount': 10,
+ 'permanentcount': 5})
+ keys = sorted(c)
+ for x in range(10):
+ self.assertEquals(x, keys[x])
+
+ def test_items(self):
+ """
+ Tests the iterating on key-value couples in the cache
+ """
+ c = QueryCache(ceiling=20)
+ # set 10 values
+ for x in range(10):
+ c[x] = x
+ # arrange for the first 5 to be permanent
+ for x in range(5):
+ for r in range(QueryCache._maxlevel + 2):
+ v = c[x]
+ self.assertEqual(v, x)
+ self.assertEqual(c._usage_report(),
+ {'transientcount': 0,
+ 'itemcount': 10,
+ 'permanentcount': 5})
+ content = sorted(c.items())
+ for x in range(10):
+ self.assertEquals(x, content[x][0])
+ self.assertEquals(x, content[x][1])
+
+
class UStringIOTC(TestCase):
def test_boolean_value(self):
self.assertTrue(UStringIO())
--- a/cubicweb/utils.py Fri Mar 08 14:35:59 2019 +0100
+++ b/cubicweb/utils.py Mon Mar 11 14:34:14 2019 +0100
@@ -632,6 +632,29 @@
with self._lock:
return len(self._data)
+ def items(self):
+ """Get an iterator over the dictionary's items: (key, value) pairs"""
+ with self._lock:
+ for k, v in self._data.items():
+ yield k, v
+
+ def get(self, k, default=None):
+ """Get the value associated to the specified key
+
+ :param k: The key to look for
+ :param default: The default value when the key is not found
+ :return: The associated value (or the default value)
+ """
+ try:
+ return self._data[k]
+ except KeyError:
+ return default
+
+ def __iter__(self):
+ with self._lock:
+ for k in iter(self._data):
+ yield k
+
def __getitem__(self, k):
with self._lock:
if k in self._permanent:
@@ -689,10 +712,20 @@
break
level = v
else:
- # we removed cruft but everything is permanent
+ # we removed cruft
if len(self._data) >= self._max:
- logger.warning('Cache %s is full.' % id(self))
- self._clear()
+ if len(self._permanent) >= self._max:
+ # we really are full with permanents => clear
+ logger.warning('Cache %s is full.' % id(self))
+ self._clear()
+ else:
+ # pathological case where _transient was probably empty ...
+ # drop all non-permanents
+ to_drop = set(self._data.keys()).difference(self._permanent)
+ for k in to_drop:
+ # should not be in _transient
+ assert k not in self._transient
+ self._data.pop(k, None)
def _usage_report(self):
with self._lock:
--- a/debian/control Fri Mar 08 14:35:59 2019 +0100
+++ b/debian/control Mon Mar 11 14:34:14 2019 +0100
@@ -28,6 +28,7 @@
python-passlib,
python-repoze.lru,
python-wsgicors,
+ python-filelock,
sphinx-common,
Standards-Version: 3.9.6
Homepage: https://www.cubicweb.org
@@ -51,6 +52,7 @@
python-passlib,
python-tz,
graphviz,
+ python-filelock,
gettext,
Recommends:
cubicweb-ctl (= ${source:Version}),
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/doc/changes/3.27.rst Mon Mar 11 14:34:14 2019 +0100
@@ -0,0 +1,15 @@
+3.27 (not yet released)
+=======================
+
+New features
+------------
+
+* Tests can now be run concurrently across multiple processes. You can use
+ `pytest-xdist`_ for that. For tests using `PostgresApptestConfiguration` you
+ should be aware that `startpgcluster()` can't run concurrently. Workaround is
+ to call pytest with ``--dist=loadfile`` to use a single test process per test
+ module or use an existing database cluster and set ``db-host`` and
+ ``db-port`` of ``devtools.DEFAULT_PSQL_SOURCES['system']`` accordingly.
+
+.. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist
+
--- a/flake8-ok-files.txt Fri Mar 08 14:35:59 2019 +0100
+++ b/flake8-ok-files.txt Mon Mar 11 14:34:14 2019 +0100
@@ -108,6 +108,7 @@
cubicweb/test/unittest_rset.py
cubicweb/test/unittest_rtags.py
cubicweb/test/unittest_schema.py
+cubicweb/test/unittest_statsd.py
cubicweb/test/unittest_toolsutils.py
cubicweb/test/unittest_wfutils.py
cubicweb/toolsutils.py
--- a/setup.py Fri Mar 08 14:35:59 2019 +0100
+++ b/setup.py Mon Mar 11 14:34:14 2019 +0100
@@ -74,6 +74,7 @@
'pytz',
'Markdown',
'unittest2 >= 0.7.0',
+ 'filelock',
],
entry_points={
'console_scripts': [