# HG changeset patch # User Denis Laxalde # Date 1552311254 -3600 # Node ID 540904e0ff0fadafec02ed7d5e49e3fcc5a17c89 # Parent 3eddb59541359cedf5f054058485a0984a4aaa94# Parent 6311651c355f31218691892d6936afe6a2c9d2a9 Merge with branch 3.26 diff -r 6311651c355f -r 540904e0ff0f cubicweb/__pkginfo__.py --- a/cubicweb/__pkginfo__.py Fri Mar 08 14:35:59 2019 +0100 +++ b/cubicweb/__pkginfo__.py Mon Mar 11 14:34:14 2019 +0100 @@ -22,8 +22,8 @@ modname = distname = "cubicweb" -numversion = (3, 26, 8) -version = '.'.join(str(num) for num in numversion) +numversion = (3, 27, 0) +version = '.'.join(str(num) for num in numversion) + '.dev0' description = "a repository of entities / relations for knowledge management" author = "Logilab" diff -r 6311651c355f -r 540904e0ff0f cubicweb/devtools/__init__.py --- a/cubicweb/devtools/__init__.py Fri Mar 08 14:35:59 2019 +0100 +++ b/cubicweb/devtools/__init__.py Mon Mar 11 14:34:14 2019 +0100 @@ -32,6 +32,7 @@ from os.path import abspath, join, exists, split, isdir, dirname from functools import partial +import filelock from six import text_type from six.moves import cPickle as pickle @@ -427,11 +428,12 @@ raise ValueError('no initialization function for driver %r' % self.DRIVER) def has_cache(self, db_id): - """Check if a given database id exist in cb cache for the current config""" - cache_glob = self.absolute_backup_file('*', '*') - if cache_glob not in self.explored_glob: - self.discover_cached_db() - return self.db_cache_key(db_id) in self.db_cache + """Check if a given database id exist in db cache for the current config""" + key = self.db_cache_key(db_id) + if key in self.db_cache: + return True + self.discover_cached_db() + return key in self.db_cache def discover_cached_db(self): """Search available db_if for the current config""" @@ -469,20 +471,23 @@ ``pre_setup_func`` to setup the database. This function backup any database it build""" - if self.has_cache(test_db_id): - return # test_db_id, 'already in cache' - if test_db_id is DEFAULT_EMPTY_DB_ID: - self.init_test_database() - else: - print('Building %s for database %s' % (test_db_id, self.dbname)) - self.build_db_cache(DEFAULT_EMPTY_DB_ID) - self.restore_database(DEFAULT_EMPTY_DB_ID) - self.get_repo(startup=True) - cnx = self.get_cnx() - with cnx: - pre_setup_func(cnx, self.config) - cnx.commit() - self.backup_database(test_db_id) + lockfile = join(self._ensure_test_backup_db_dir(), + '{}.lock'.format(test_db_id)) + with filelock.FileLock(lockfile): + if self.has_cache(test_db_id): + return # test_db_id, 'already in cache' + if test_db_id is DEFAULT_EMPTY_DB_ID: + self.init_test_database() + else: + print('Building %s for database %s' % (test_db_id, self.dbname)) + self.build_db_cache(DEFAULT_EMPTY_DB_ID) + self.restore_database(DEFAULT_EMPTY_DB_ID) + self.get_repo(startup=True) + cnx = self.get_cnx() + with cnx: + pre_setup_func(cnx, self.config) + cnx.commit() + self.backup_database(test_db_id) class NoCreateDropDatabaseHandler(TestDataBaseHandler): diff -r 6311651c355f -r 540904e0ff0f cubicweb/pyramid/core.py diff -r 6311651c355f -r 540904e0ff0f cubicweb/rset.py --- a/cubicweb/rset.py Fri Mar 08 14:35:59 2019 +0100 +++ b/cubicweb/rset.py Mon Mar 11 14:34:14 2019 +0100 @@ -1,4 +1,4 @@ -# copyright 2003-2016 LOGILAB S.A. (Paris, FRANCE), all rights reserved. +# copyright 2003-2018 LOGILAB S.A. (Paris, FRANCE), all rights reserved. # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr # # This file is part of CubicWeb. @@ -393,6 +393,8 @@ if self.rows[i][col] is not None: yield self.get_entity(i, col) + all = entities + def iter_rows_with_entities(self): """ iterates over rows, and for each row eids are converted to plain entities @@ -459,6 +461,34 @@ else: raise MultipleResultsError("Multiple rows were found for one()") + def first(self, col=0): + """Retrieve the first entity from the query. + + If the result set is empty, raises :exc:`NoResultError`. + + :type col: int + :param col: The column localising the entity in the unique row + + :return: the partially initialized `Entity` instance + """ + if len(self) == 0: + raise NoResultError("No row was found for first()") + return self.get_entity(0, col) + + def last(self, col=0): + """Retrieve the last entity from the query. + + If the result set is empty, raises :exc:`NoResultError`. + + :type col: int + :param col: The column localising the entity in the unique row + + :return: the partially initialized `Entity` instance + """ + if len(self) == 0: + raise NoResultError("No row was found for last()") + return self.get_entity(-1, col) + def _make_entity(self, row, col): """Instantiate an entity, and store it in the entity cache""" # build entity instance diff -r 6311651c355f -r 540904e0ff0f cubicweb/statsd_logger.py --- a/cubicweb/statsd_logger.py Fri Mar 08 14:35:59 2019 +0100 +++ b/cubicweb/statsd_logger.py Mon Mar 11 14:34:14 2019 +0100 @@ -58,6 +58,7 @@ import time import socket +from contextlib import contextmanager _bucket = 'cubicweb' _address = None @@ -87,19 +88,32 @@ _socket = socket.socket(family, socket.SOCK_DGRAM) +def teardown(): + """Unconfigure the statsd endpoint + + This is most likely only useful for unit tests""" + global _bucket, _address, _socket + _bucket = 'cubicweb' + _address = None + _socket = None + + def statsd_c(context, n=1): if _address is not None: - _socket.sendto('{0}.{1}:{2}|c\n'.format(_bucket, context, n), _address) + _socket.sendto('{0}.{1}:{2}|c\n'.format(_bucket, context, n).encode(), + _address) def statsd_g(context, value): if _address is not None: - _socket.sendto('{0}.{1}:{2}|g\n'.format(_bucket, context, value), _address) + _socket.sendto('{0}.{1}:{2}|g\n'.format(_bucket, context, value).encode(), + _address) def statsd_t(context, value): if _address is not None: - _socket.sendto('{0}.{1}:{2:.4f}|ms\n'.format(_bucket, context, value), _address) + _socket.sendto('{0}.{1}:{2:.4f}|ms\n'.format(_bucket, context, value).encode(), + _address) class statsd_timeit(object): @@ -125,7 +139,7 @@ finally: dt = 1000 * (time.time() - t0) msg = '{0}.{1}:{2:.4f}|ms\n{0}.{1}:1|c\n'.format( - _bucket, self.__name__, dt) + _bucket, self.__name__, dt).encode() _socket.sendto(msg, _address) def __get__(self, obj, objtype): @@ -134,3 +148,17 @@ return self import functools return functools.partial(self.__call__, obj) + + +@contextmanager +def statsd_timethis(ctxmsg): + if _address is not None: + t0 = time.time() + try: + yield + finally: + if _address is not None: + dt = 1000 * (time.time() - t0) + msg = '{0}.{1}:{2:.4f}|ms\n{0}.{1}:1|c\n'.format( + _bucket, ctxmsg, dt).encode() + _socket.sendto(msg, _address) diff -r 6311651c355f -r 540904e0ff0f cubicweb/test/unittest_rset.py --- a/cubicweb/test/unittest_rset.py Fri Mar 08 14:35:59 2019 +0100 +++ b/cubicweb/test/unittest_rset.py Mon Mar 11 14:34:14 2019 +0100 @@ -1,5 +1,5 @@ # coding: utf-8 -# copyright 2003-2016 LOGILAB S.A. (Paris, FRANCE), all rights reserved. +# copyright 2003-2018 LOGILAB S.A. (Paris, FRANCE), all rights reserved. # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr # # This file is part of CubicWeb. @@ -444,6 +444,78 @@ with self.assertRaises(MultipleResultsError): req.execute('Any X WHERE X is CWUser').one() + def test_first(self): + with self.admin_access.web_request() as req: + req.create_entity('CWUser', + login=u'cdevienne', + upassword=u'cdevienne', + surname=u'de Vienne', + firstname=u'Christophe') + e = req.execute('Any X WHERE X login "cdevienne"').first() + self.assertEqual(e.surname, u'de Vienne') + + e = req.execute( + 'Any X, N WHERE X login "cdevienne", X surname N').first() + self.assertEqual(e.surname, u'de Vienne') + + e = req.execute( + 'Any N, X WHERE X login "cdevienne", X surname N').first(col=1) + self.assertEqual(e.surname, u'de Vienne') + + def test_first_no_rows(self): + with self.admin_access.web_request() as req: + with self.assertRaises(NoResultError): + req.execute('Any X WHERE X login "patanok"').first() + + def test_first_multiple_rows(self): + with self.admin_access.web_request() as req: + req.create_entity( + 'CWUser', login=u'user1', upassword=u'cdevienne', + surname=u'de Vienne', firstname=u'Christophe') + req.create_entity( + 'CWUser', login=u'user2', upassword='adim', + surname=u'di mascio', firstname=u'adrien') + + e = req.execute('Any X ORDERBY X WHERE X is CWUser, ' + 'X login LIKE "user%"').first() + self.assertEqual(e.login, 'user1') + + def test_last(self): + with self.admin_access.web_request() as req: + req.create_entity('CWUser', + login=u'cdevienne', + upassword=u'cdevienne', + surname=u'de Vienne', + firstname=u'Christophe') + e = req.execute('Any X WHERE X login "cdevienne"').last() + self.assertEqual(e.surname, u'de Vienne') + + e = req.execute( + 'Any X, N WHERE X login "cdevienne", X surname N').last() + self.assertEqual(e.surname, u'de Vienne') + + e = req.execute( + 'Any N, X WHERE X login "cdevienne", X surname N').last(col=1) + self.assertEqual(e.surname, u'de Vienne') + + def test_last_no_rows(self): + with self.admin_access.web_request() as req: + with self.assertRaises(NoResultError): + req.execute('Any X WHERE X login "patanok"').last() + + def test_last_multiple_rows(self): + with self.admin_access.web_request() as req: + req.create_entity( + 'CWUser', login=u'user1', upassword=u'cdevienne', + surname=u'de Vienne', firstname=u'Christophe') + req.create_entity( + 'CWUser', login=u'user2', upassword='adim', + surname=u'di mascio', firstname=u'adrien') + + e = req.execute('Any X ORDERBY X WHERE X is CWUser, ' + 'X login LIKE "user%"').last() + self.assertEqual(e.login, 'user2') + def test_related_entity_optional(self): with self.admin_access.web_request() as req: req.create_entity('Bookmark', title=u'aaaa', path=u'path') diff -r 6311651c355f -r 540904e0ff0f cubicweb/test/unittest_statsd.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cubicweb/test/unittest_statsd.py Mon Mar 11 14:34:14 2019 +0100 @@ -0,0 +1,136 @@ +# -*- coding: utf-8 -*- +# copyright 2018 LOGILAB S.A. (Paris, FRANCE), all rights reserved. +# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr +# +# This file is part of CubicWeb. +# +# CubicWeb is free software: you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation, either version 2.1 of the License, or (at your option) +# any later version. +# +# CubicWeb is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License along +# with CubicWeb. If not, see . +"""unit tests for module cubicweb.statsd_logger""" + +import threading +import socket +import time +import re + +from unittest import TestCase +from cubicweb import statsd_logger as statsd + + +UDP_PORT = None +RUNNING = True +SOCK = socket.socket(socket.AF_INET, + socket.SOCK_DGRAM) +SOCK.settimeout(0.1) +STATSD = None +DATA = [] + + +def statsd_rcv(): + while RUNNING: + try: + data, addr = SOCK.recvfrom(1024) + if data: + rcv = [row.strip().decode() for row in data.splitlines()] + DATA.extend(rcv) + except socket.timeout: + pass + + +def setUpModule(*args): + global UDP_PORT, STATSD + SOCK.bind(('127.0.0.1', 0)) + UDP_PORT = SOCK.getsockname()[1] + STATSD = threading.Thread(target=statsd_rcv) + STATSD.start() + statsd.setup('test', ('127.0.0.1', UDP_PORT)) + + +def tearDownModule(*args): + global RUNNING + RUNNING = False + STATSD.join() + statsd.teardown() + + +class StatsdTC(TestCase): + + def setUp(self): + super(StatsdTC, self).setUp() + DATA[:] = [] + + def check_received(self, value): + for i in range(10): + if value in DATA: + break + time.sleep(0.01) + else: + self.assertIn(value, DATA) + + def check_received_ms(self, value): + value = re.compile(value.replace('?', r'\d')) + for i in range(10): + if [x for x in DATA if value.match(x)]: + break + time.sleep(0.01) + else: + self.assertTrue([x for x in DATA if value.match(x)], DATA) + + def test_statsd_c(self): + statsd.statsd_c('context') + self.check_received('test.context:1|c') + statsd.statsd_c('context', 10) + self.check_received('test.context:10|c') + + def test_statsd_g(self): + statsd.statsd_g('context', 42) + self.check_received('test.context:42|g') + statsd.statsd_g('context', 'Igorrr') + self.check_received('test.context:Igorrr|g') + + def test_statsd_t(self): + statsd.statsd_t('context', 1) + self.check_received('test.context:1.0000|ms') + statsd.statsd_t('context', 10) + self.check_received('test.context:10.0000|ms') + statsd.statsd_t('context', 0.12344) + self.check_received('test.context:0.1234|ms') + statsd.statsd_t('context', 0.12345) + self.check_received('test.context:0.1235|ms') + + def test_decorator(self): + + @statsd.statsd_timeit + def measure_me_please(): + "some nice function" + return 42 + + self.assertEqual(measure_me_please.__doc__, + "some nice function") + + measure_me_please() + self.check_received_ms('test.measure_me_please:0.0???|ms') + self.check_received('test.measure_me_please:1|c') + + def test_context_manager(self): + + with statsd.statsd_timethis('cm'): + time.sleep(0.1) + + self.check_received_ms('test.cm:100.????|ms') + self.check_received('test.cm:1|c') + + +if __name__ == '__main__': + from unittest import main + main() diff -r 6311651c355f -r 540904e0ff0f cubicweb/test/unittest_utils.py --- a/cubicweb/test/unittest_utils.py Fri Mar 08 14:35:59 2019 +0100 +++ b/cubicweb/test/unittest_utils.py Mon Mar 11 14:34:14 2019 +0100 @@ -109,6 +109,95 @@ 'itemcount': 10, 'permanentcount': 5}) + def test_clear_on_overflow(self): + """Tests that only non-permanent items in the cache are wiped-out on ceiling overflow + """ + c = QueryCache(ceiling=10) + # set 10 values + for x in range(10): + c[x] = x + # arrange for the first 5 to be permanent + for x in range(5): + for r in range(QueryCache._maxlevel + 2): + v = c[x] + self.assertEqual(v, x) + # Add the 11-th + c[10] = 10 + self.assertEqual(c._usage_report(), + {'transientcount': 0, + 'itemcount': 6, + 'permanentcount': 5}) + + def test_get_with_default(self): + """ + Tests the capability of QueryCache for retrieving items with a default value + """ + c = QueryCache(ceiling=20) + # set 10 values + for x in range(10): + c[x] = x + # arrange for the first 5 to be permanent + for x in range(5): + for r in range(QueryCache._maxlevel + 2): + v = c[x] + self.assertEqual(v, x) + self.assertEqual(c._usage_report(), + {'transientcount': 0, + 'itemcount': 10, + 'permanentcount': 5}) + # Test defaults for existing (including in permanents) + for x in range(10): + v = c.get(x, -1) + self.assertEqual(v, x) + # Test defaults for others + for x in range(10, 15): + v = c.get(x, -1) + self.assertEqual(v, -1) + + def test_iterkeys(self): + """ + Tests the iterating on keys in the cache + """ + c = QueryCache(ceiling=20) + # set 10 values + for x in range(10): + c[x] = x + # arrange for the first 5 to be permanent + for x in range(5): + for r in range(QueryCache._maxlevel + 2): + v = c[x] + self.assertEqual(v, x) + self.assertEqual(c._usage_report(), + {'transientcount': 0, + 'itemcount': 10, + 'permanentcount': 5}) + keys = sorted(c) + for x in range(10): + self.assertEquals(x, keys[x]) + + def test_items(self): + """ + Tests the iterating on key-value couples in the cache + """ + c = QueryCache(ceiling=20) + # set 10 values + for x in range(10): + c[x] = x + # arrange for the first 5 to be permanent + for x in range(5): + for r in range(QueryCache._maxlevel + 2): + v = c[x] + self.assertEqual(v, x) + self.assertEqual(c._usage_report(), + {'transientcount': 0, + 'itemcount': 10, + 'permanentcount': 5}) + content = sorted(c.items()) + for x in range(10): + self.assertEquals(x, content[x][0]) + self.assertEquals(x, content[x][1]) + + class UStringIOTC(TestCase): def test_boolean_value(self): self.assertTrue(UStringIO()) diff -r 6311651c355f -r 540904e0ff0f cubicweb/utils.py --- a/cubicweb/utils.py Fri Mar 08 14:35:59 2019 +0100 +++ b/cubicweb/utils.py Mon Mar 11 14:34:14 2019 +0100 @@ -632,6 +632,29 @@ with self._lock: return len(self._data) + def items(self): + """Get an iterator over the dictionary's items: (key, value) pairs""" + with self._lock: + for k, v in self._data.items(): + yield k, v + + def get(self, k, default=None): + """Get the value associated to the specified key + + :param k: The key to look for + :param default: The default value when the key is not found + :return: The associated value (or the default value) + """ + try: + return self._data[k] + except KeyError: + return default + + def __iter__(self): + with self._lock: + for k in iter(self._data): + yield k + def __getitem__(self, k): with self._lock: if k in self._permanent: @@ -689,10 +712,20 @@ break level = v else: - # we removed cruft but everything is permanent + # we removed cruft if len(self._data) >= self._max: - logger.warning('Cache %s is full.' % id(self)) - self._clear() + if len(self._permanent) >= self._max: + # we really are full with permanents => clear + logger.warning('Cache %s is full.' % id(self)) + self._clear() + else: + # pathological case where _transient was probably empty ... + # drop all non-permanents + to_drop = set(self._data.keys()).difference(self._permanent) + for k in to_drop: + # should not be in _transient + assert k not in self._transient + self._data.pop(k, None) def _usage_report(self): with self._lock: diff -r 6311651c355f -r 540904e0ff0f debian/control --- a/debian/control Fri Mar 08 14:35:59 2019 +0100 +++ b/debian/control Mon Mar 11 14:34:14 2019 +0100 @@ -28,6 +28,7 @@ python-passlib, python-repoze.lru, python-wsgicors, + python-filelock, sphinx-common, Standards-Version: 3.9.6 Homepage: https://www.cubicweb.org @@ -51,6 +52,7 @@ python-passlib, python-tz, graphviz, + python-filelock, gettext, Recommends: cubicweb-ctl (= ${source:Version}), diff -r 6311651c355f -r 540904e0ff0f doc/changes/3.27.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/doc/changes/3.27.rst Mon Mar 11 14:34:14 2019 +0100 @@ -0,0 +1,15 @@ +3.27 (not yet released) +======================= + +New features +------------ + +* Tests can now be run concurrently across multiple processes. You can use + `pytest-xdist`_ for that. For tests using `PostgresApptestConfiguration` you + should be aware that `startpgcluster()` can't run concurrently. Workaround is + to call pytest with ``--dist=loadfile`` to use a single test process per test + module or use an existing database cluster and set ``db-host`` and + ``db-port`` of ``devtools.DEFAULT_PSQL_SOURCES['system']`` accordingly. + +.. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist + diff -r 6311651c355f -r 540904e0ff0f doc/tutorials/base/blog-in-five-minutes.rst diff -r 6311651c355f -r 540904e0ff0f flake8-ok-files.txt --- a/flake8-ok-files.txt Fri Mar 08 14:35:59 2019 +0100 +++ b/flake8-ok-files.txt Mon Mar 11 14:34:14 2019 +0100 @@ -108,6 +108,7 @@ cubicweb/test/unittest_rset.py cubicweb/test/unittest_rtags.py cubicweb/test/unittest_schema.py +cubicweb/test/unittest_statsd.py cubicweb/test/unittest_toolsutils.py cubicweb/test/unittest_wfutils.py cubicweb/toolsutils.py diff -r 6311651c355f -r 540904e0ff0f requirements/test-web.txt diff -r 6311651c355f -r 540904e0ff0f setup.py --- a/setup.py Fri Mar 08 14:35:59 2019 +0100 +++ b/setup.py Mon Mar 11 14:34:14 2019 +0100 @@ -74,6 +74,7 @@ 'pytz', 'Markdown', 'unittest2 >= 0.7.0', + 'filelock', ], entry_points={ 'console_scripts': [ diff -r 6311651c355f -r 540904e0ff0f tox.ini