| author | Denis Laxalde <denis.laxalde@logilab.fr> |
| Tue, 19 Jun 2018 09:13:40 +0200 | |
| changeset 12327 | 58f05ffafeca |
| parent 12318 | e947954e0ffc (diff) |
| parent 12326 | 034bca253b55 (current diff) |
| child 12328 | b570d3094e32 |
--- a/cubicweb/__pkginfo__.py Mon Jun 18 10:04:08 2018 +0200 +++ b/cubicweb/__pkginfo__.py Tue Jun 19 09:13:40 2018 +0200 @@ -22,8 +22,8 @@ modname = distname = "cubicweb" -numversion = (3, 26, 4) -version = '.'.join(str(num) for num in numversion) +numversion = (3, 27, 0) +version = '.'.join(str(num) for num in numversion) + '.dev0' description = "a repository of entities / relations for knowledge management" author = "Logilab"
--- a/cubicweb/rset.py Mon Jun 18 10:04:08 2018 +0200 +++ b/cubicweb/rset.py Tue Jun 19 09:13:40 2018 +0200 @@ -1,4 +1,4 @@ -# copyright 2003-2016 LOGILAB S.A. (Paris, FRANCE), all rights reserved. +# copyright 2003-2018 LOGILAB S.A. (Paris, FRANCE), all rights reserved. # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr # # This file is part of CubicWeb. @@ -393,6 +393,8 @@ if self.rows[i][col] is not None: yield self.get_entity(i, col) + all = entities + def iter_rows_with_entities(self): """ iterates over rows, and for each row eids are converted to plain entities @@ -459,6 +461,34 @@ else: raise MultipleResultsError("Multiple rows were found for one()") + def first(self, col=0): + """Retrieve the first entity from the query. + + If the result set is empty, raises :exc:`NoResultError`. + + :type col: int + :param col: The column localising the entity in the unique row + + :return: the partially initialized `Entity` instance + """ + if len(self) == 0: + raise NoResultError("No row was found for first()") + return self.get_entity(0, col) + + def last(self, col=0): + """Retrieve the last entity from the query. + + If the result set is empty, raises :exc:`NoResultError`. + + :type col: int + :param col: The column localising the entity in the unique row + + :return: the partially initialized `Entity` instance + """ + if len(self) == 0: + raise NoResultError("No row was found for last()") + return self.get_entity(-1, col) + def _make_entity(self, row, col): """Instantiate an entity, and store it in the entity cache""" # build entity instance
--- a/cubicweb/server/edition.py Mon Jun 18 10:04:08 2018 +0200 +++ b/cubicweb/server/edition.py Tue Jun 19 09:13:40 2018 +0200 @@ -155,5 +155,6 @@ thecopy = EditedEntity(copy(self.entity)) thecopy.entity.cw_attr_cache = copy(self.entity.cw_attr_cache) thecopy.entity._cw_related_cache = {} + thecopy.entity._cw_adapters_cache = {} thecopy.update(self, skipsec=False) return thecopy
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cubicweb/server/test/unittest_edition.py Tue Jun 19 09:13:40 2018 +0200 @@ -0,0 +1,57 @@ +# copyright 2018 LOGILAB S.A. (Paris, FRANCE), all rights reserved. +# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr +# +# This file is part of CubicWeb. +# +# CubicWeb is free software: you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation, either version 2.1 of the License, or (at your option) +# any later version. +# +# CubicWeb is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License along +# with CubicWeb. If not, see <http://www.gnu.org/licenses/>. + +"""Tests for the entity edition""" + +from cubicweb.devtools.testlib import CubicWebTC +from cubicweb.server.edition import EditedEntity + + +class EditedEntityTC(CubicWebTC): + """ + Test cases for EditedEntity + """ + + def test_clone_cache_reset(self): + """ + Tests that when an EditedEntity is cloned the caches are reset in the cloned instance + :return: Nothing + """ + # Create an entity, create the EditedEntity and clone it + with self.admin_access.cnx() as cnx: + affaire = cnx.create_entity("Affaire", sujet=u"toto") + ee = EditedEntity(affaire) + ee.entity.cw_adapt_to("IWorkflowable") + self.assertTrue(ee.entity._cw_related_cache) + self.assertTrue(ee.entity._cw_adapters_cache) + the_clone = ee.clone() + self.assertFalse(the_clone.entity._cw_related_cache) + self.assertFalse(the_clone.entity._cw_adapters_cache) + cnx.rollback() + # Check the attributes + with self.admin_access.cnx() as cnx: + # Assume a different connection set on the entity + self.assertNotEqual(the_clone.entity._cw, cnx) + # Use the new connection + the_clone.entity._cw = cnx + self.assertEqual("toto", the_clone.entity.sujet) + + +if __name__ == '__main__': + import unittest + unittest.main()
--- a/cubicweb/statsd_logger.py Mon Jun 18 10:04:08 2018 +0200 +++ b/cubicweb/statsd_logger.py Tue Jun 19 09:13:40 2018 +0200 @@ -58,6 +58,7 @@ import time import socket +from contextlib import contextmanager _bucket = 'cubicweb' _address = None @@ -87,19 +88,32 @@ _socket = socket.socket(family, socket.SOCK_DGRAM) +def teardown(): + """Unconfigure the statsd endpoint + + This is most likely only useful for unit tests""" + global _bucket, _address, _socket + _bucket = 'cubicweb' + _address = None + _socket = None + + def statsd_c(context, n=1): if _address is not None: - _socket.sendto('{0}.{1}:{2}|c\n'.format(_bucket, context, n), _address) + _socket.sendto('{0}.{1}:{2}|c\n'.format(_bucket, context, n).encode(), + _address) def statsd_g(context, value): if _address is not None: - _socket.sendto('{0}.{1}:{2}|g\n'.format(_bucket, context, value), _address) + _socket.sendto('{0}.{1}:{2}|g\n'.format(_bucket, context, value).encode(), + _address) def statsd_t(context, value): if _address is not None: - _socket.sendto('{0}.{1}:{2:.4f}|ms\n'.format(_bucket, context, value), _address) + _socket.sendto('{0}.{1}:{2:.4f}|ms\n'.format(_bucket, context, value).encode(), + _address) class statsd_timeit(object): @@ -125,7 +139,7 @@ finally: dt = 1000 * (time.time() - t0) msg = '{0}.{1}:{2:.4f}|ms\n{0}.{1}:1|c\n'.format( - _bucket, self.__name__, dt) + _bucket, self.__name__, dt).encode() _socket.sendto(msg, _address) def __get__(self, obj, objtype): @@ -134,3 +148,17 @@ return self import functools return functools.partial(self.__call__, obj) + + +@contextmanager +def statsd_timethis(ctxmsg): + if _address is not None: + t0 = time.time() + try: + yield + finally: + if _address is not None: + dt = 1000 * (time.time() - t0) + msg = '{0}.{1}:{2:.4f}|ms\n{0}.{1}:1|c\n'.format( + _bucket, ctxmsg, dt).encode() + _socket.sendto(msg, _address)
--- a/cubicweb/test/unittest_rset.py Mon Jun 18 10:04:08 2018 +0200 +++ b/cubicweb/test/unittest_rset.py Tue Jun 19 09:13:40 2018 +0200 @@ -1,5 +1,5 @@ # coding: utf-8 -# copyright 2003-2016 LOGILAB S.A. (Paris, FRANCE), all rights reserved. +# copyright 2003-2018 LOGILAB S.A. (Paris, FRANCE), all rights reserved. # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr # # This file is part of CubicWeb. @@ -16,7 +16,7 @@ # # You should have received a copy of the GNU Lesser General Public License along # with CubicWeb. If not, see <http://www.gnu.org/licenses/>. -"""unit tests for module cubicweb.utils""" +"""unit tests for module cubicweb.rset""" from six import string_types from six.moves import cPickle as pickle @@ -44,9 +44,12 @@ def test_relations_description(self): """tests relations_description() function""" queries = { - 'Any U,L,M where U is CWUser, U login L, U mail M': [(1, 'login', 'subject'), (2, 'mail', 'subject')], - 'Any U,L,M where U is CWUser, L is Foo, U mail M': [(2, 'mail', 'subject')], - 'Any C,P where C is Company, C employs P': [(1, 'employs', 'subject')], + 'Any U,L,M where U is CWUser, U login L, U mail M': [ + (1, 'login', 'subject'), (2, 'mail', 'subject')], + 'Any U,L,M where U is CWUser, L is Foo, U mail M': [ + (2, 'mail', 'subject')], + 'Any C,P where C is Company, C employs P': [ + (1, 'employs', 'subject')], 'Any C,P where C is Company, P employed_by P': [], 'Any C where C is Company, C employs P': [], } @@ -68,14 +71,17 @@ def test_subquery_callfunc(self): rql = ('Any A,B,C,COUNT(D) GROUPBY A,B,C WITH A,B,C,D BEING ' - '(Any YEAR(CD), MONTH(CD), S, X WHERE X is CWUser, X creation_date CD, X in_state S)') + '(Any YEAR(CD), MONTH(CD), S, X WHERE X is CWUser, ' + 'X creation_date CD, X in_state S)') rqlst = parse(rql) select, col = rqlst.locate_subquery(2, 'CWUser', None) result = list(attr_desc_iterator(select, col, 2)) self.assertEqual(result, []) def test_subquery_callfunc_2(self): - rql = ('Any X,S,L WHERE X in_state S WITH X, L BEING (Any X,MAX(L) GROUPBY X WHERE X is CWUser, T wf_info_for X, T creation_date L)') + rql = ('Any X,S,L WHERE ' + 'X in_state S WITH X, L BEING (Any X,MAX(L) GROUPBY X WHERE ' + 'X is CWUser, T wf_info_for X, T creation_date L)') rqlst = parse(rql) select, col = rqlst.locate_subquery(0, 'CWUser', None) result = list(attr_desc_iterator(select, col, 0)) @@ -115,20 +121,19 @@ '%sview?vid=foo&rql=yo' % baseurl) self.compare_urls(req.build_url('view', _restpath='task/title/go'), '%stask/title/go' % baseurl) - #self.compare_urls(req.build_url('view', _restpath='/task/title/go'), - # '%stask/title/go' % baseurl) + # self.compare_urls(req.build_url('view', _restpath='/task/title/go'), + # '%stask/title/go' % baseurl) # empty _restpath should not crash self.compare_urls(req.build_url('view', _restpath=''), baseurl) - def test_build(self): """test basic build of a ResultSet""" - rs = ResultSet([1,2,3], 'CWGroup X', description=['CWGroup', 'CWGroup', 'CWGroup']) + rs = ResultSet([1, 2, 3], 'CWGroup X', + description=['CWGroup', 'CWGroup', 'CWGroup']) self.assertEqual(rs.rowcount, 3) - self.assertEqual(rs.rows, [1,2,3]) + self.assertEqual(rs.rows, [1, 2, 3]) self.assertEqual(rs.description, ['CWGroup', 'CWGroup', 'CWGroup']) - def test_limit(self): rs = ResultSet([[12000, 'adim'], [13000, 'syt'], [14000, 'nico']], 'Any U,L where U is CWUser, U login L', @@ -148,7 +153,7 @@ rs = req.execute('Any E,U WHERE E is CWEType, E created_by U') # get entity on row 9. This will fill its created_by relation cache, # with cwuser on row 9 as well - e1 = rs.get_entity(9, 0) + e1 = rs.get_entity(9, 0) # noqa # get entity on row 10. This will fill its created_by relation cache, # with cwuser built on row 9 e2 = rs.get_entity(10, 0) @@ -171,6 +176,7 @@ with self.admin_access.web_request() as req: rs.req = req rs.vreg = self.vreg + def test_filter(entity): return entity.login != 'nico' @@ -185,12 +191,13 @@ description=[['CWUser', 'String']] * 3) with self.admin_access.web_request() as req: rs.req = req + def test_transform(row, desc): return row[1:], desc[1:] rs2 = rs.transformed_rset(test_transform) self.assertEqual(len(rs2), 3) - self.assertEqual(list(rs2), [['adim'],['syt'],['nico']]) + self.assertEqual(list(rs2), [['adim'], ['syt'], ['nico']]) def test_sort(self): rs = ResultSet([[12000, 'adim'], [13000, 'syt'], [14000, 'nico']], @@ -200,13 +207,13 @@ rs.req = req rs.vreg = self.vreg - rs2 = rs.sorted_rset(lambda e:e.cw_attr_cache['login']) + rs2 = rs.sorted_rset(lambda e: e.cw_attr_cache['login']) self.assertEqual(len(rs2), 3) self.assertEqual([login for _, login in rs2], ['adim', 'nico', 'syt']) # make sure rs is unchanged self.assertEqual([login for _, login in rs], ['adim', 'syt', 'nico']) - rs2 = rs.sorted_rset(lambda e:e.cw_attr_cache['login'], reverse=True) + rs2 = rs.sorted_rset(lambda e: e.cw_attr_cache['login'], reverse=True) self.assertEqual(len(rs2), 3) self.assertEqual([login for _, login in rs2], ['syt', 'nico', 'adim']) # make sure rs is unchanged @@ -221,47 +228,47 @@ def test_split(self): rs = ResultSet([[12000, 'adim', u'Adim chez les pinguins'], [12000, 'adim', u'Jardiner facile'], - [13000, 'syt', u'Le carrelage en 42 leçons'], + [13000, 'syt', u'Le carrelage en 42 leçons'], [14000, 'nico', u'La tarte tatin en 15 minutes'], [14000, 'nico', u"L'épluchage du castor commun"]], - 'Any U, L, T WHERE U is CWUser, U login L,'\ - 'D created_by U, D title T', + ('Any U, L, T WHERE U is CWUser, U login L,' + 'D created_by U, D title T'), description=[['CWUser', 'String', 'String']] * 5) with self.admin_access.web_request() as req: rs.req = req rs.vreg = self.vreg - rsets = rs.split_rset(lambda e:e.cw_attr_cache['login']) + rsets = rs.split_rset(lambda e: e.cw_attr_cache['login']) self.assertEqual(len(rsets), 3) - self.assertEqual([login for _, login,_ in rsets[0]], ['adim', 'adim']) - self.assertEqual([login for _, login,_ in rsets[1]], ['syt']) - self.assertEqual([login for _, login,_ in rsets[2]], ['nico', 'nico']) + self.assertEqual([login for _, login, _ in rsets[0]], ['adim', 'adim']) + self.assertEqual([login for _, login, _ in rsets[1]], ['syt']) + self.assertEqual([login for _, login, _ in rsets[2]], ['nico', 'nico']) # make sure rs is unchanged - self.assertEqual([login for _, login,_ in rs], ['adim', 'adim', 'syt', 'nico', 'nico']) + self.assertEqual([login for _, login, _ in rs], ['adim', 'adim', 'syt', 'nico', 'nico']) - rsets = rs.split_rset(lambda e:e.cw_attr_cache['login'], return_dict=True) + rsets = rs.split_rset(lambda e: e.cw_attr_cache['login'], return_dict=True) self.assertEqual(len(rsets), 3) - self.assertEqual([login for _, login,_ in rsets['nico']], ['nico', 'nico']) - self.assertEqual([login for _, login,_ in rsets['adim']], ['adim', 'adim']) - self.assertEqual([login for _, login,_ in rsets['syt']], ['syt']) + self.assertEqual([login for _, login, _ in rsets['nico']], ['nico', 'nico']) + self.assertEqual([login for _, login, _ in rsets['adim']], ['adim', 'adim']) + self.assertEqual([login for _, login, _ in rsets['syt']], ['syt']) # make sure rs is unchanged - self.assertEqual([login for _, login,_ in rs], ['adim', 'adim', 'syt', 'nico', 'nico']) + self.assertEqual([login for _, login, _ in rs], ['adim', 'adim', 'syt', 'nico', 'nico']) rsets = rs.split_rset(lambda s: s.count('d'), col=2) self.assertEqual(len(rsets), 2) self.assertEqual([title for _, _, title in rsets[0]], - [u"Adim chez les pinguins", - u"Jardiner facile", - u"L'épluchage du castor commun",]) + [u"Adim chez les pinguins", + u"Jardiner facile", + u"L'épluchage du castor commun"]) self.assertEqual([title for _, _, title in rsets[1]], - [u"Le carrelage en 42 leçons", - u"La tarte tatin en 15 minutes",]) + [u"Le carrelage en 42 leçons", + u"La tarte tatin en 15 minutes"]) # make sure rs is unchanged self.assertEqual([title for _, _, title in rs], - [u'Adim chez les pinguins', - u'Jardiner facile', - u'Le carrelage en 42 leçons', - u'La tarte tatin en 15 minutes', - u"L'épluchage du castor commun"]) + [u'Adim chez les pinguins', + u'Jardiner facile', + u'Le carrelage en 42 leçons', + u'La tarte tatin en 15 minutes', + u"L'épluchage du castor commun"]) def test_cached_syntax_tree(self): """make sure syntax tree is cached""" @@ -308,13 +315,11 @@ self.assertEqual(e.cw_col, 1) self.assertEqual(e.cw_attr_cache['login'], 'anon') self.assertRaises(KeyError, e.cw_attr_cache.__getitem__, 'firstname') - self.assertEqual(pprelcachedict(e._cw_related_cache), - []) + self.assertEqual(pprelcachedict(e._cw_related_cache), []) e.complete() self.assertEqual(e.cw_attr_cache['firstname'], None) self.assertEqual(e.view('text'), 'anon') - self.assertEqual(pprelcachedict(e._cw_related_cache), - []) + self.assertEqual(pprelcachedict(e._cw_related_cache), []) self.assertRaises(NotAnEntity, rset.get_entity, 0, 2) self.assertRaises(NotAnEntity, rset.get_entity, 0, 3) @@ -327,14 +332,14 @@ # for_user / in_group are prefetched in CWUser __init__, in_state should # be filed from our query rset self.assertEqual(pprelcachedict(e._cw_related_cache), - [('in_state_subject', [seid])]) + [('in_state_subject', [seid])]) def test_get_entity_advanced_prefilled_cache(self): with self.admin_access.web_request() as req: e = req.create_entity('Bookmark', title=u'zou', path=u'path') req.cnx.commit() rset = req.execute('Any X,U,S,XT,UL,SN WHERE X created_by U, U in_state S, ' - 'X title XT, S name SN, U login UL, X eid %s' % e.eid) + 'X title XT, S name SN, U login UL, X eid %s' % e.eid) e = rset.get_entity(0, 0) self.assertEqual(e.cw_attr_cache['title'], 'zou') self.assertEqual(pprelcachedict(e._cw_related_cache), @@ -371,11 +376,10 @@ u = cnx.user self.assertTrue(u.cw_relation_cached('primary_email', 'subject')) - def test_get_entity_cache_with_left_outer_join(self): with self.admin_access.web_request() as req: eid = req.execute('INSERT CWUser E: E login "joe", E upassword "joe", E in_group G ' - 'WHERE G name "users"')[0][0] + 'WHERE G name "users"')[0][0] rset = req.execute('Any X,E WHERE X eid %(x)s, X primary_email E?', {'x': eid}) e = rset.get_entity(0, 0) # if any of the assertion below fails with a KeyError, the relation is not cached @@ -386,19 +390,18 @@ self.assertIsInstance(cached, ResultSet) self.assertEqual(cached.rowcount, 0) - def test_get_entity_union(self): with self.admin_access.web_request() as req: - e = req.create_entity('Bookmark', title=u'manger', path=u'path') + req.create_entity('Bookmark', title=u'manger', path=u'path') req.drop_entity_cache() rset = req.execute('Any X,N ORDERBY N WITH X,N BEING ' - '((Any X,N WHERE X is Bookmark, X title N)' - ' UNION ' - ' (Any X,N WHERE X is CWGroup, X name N))') + '((Any X,N WHERE X is Bookmark, X title N)' + ' UNION ' + ' (Any X,N WHERE X is CWGroup, X name N))') expected = (('CWGroup', 'guests'), ('CWGroup', 'managers'), ('Bookmark', 'manger'), ('CWGroup', 'owners'), ('CWGroup', 'users')) - for entity in rset.entities(): # test get_entity for each row actually + for entity in rset.entities(): # test get_entity for each row actually etype, n = expected[entity.cw_row] self.assertEqual(entity.cw_etype, etype) attr = etype == 'Bookmark' and 'title' or 'name' @@ -406,10 +409,11 @@ def test_one(self): with self.admin_access.web_request() as req: - req.create_entity('CWUser', login=u'cdevienne', - upassword=u'cdevienne', - surname=u'de Vienne', - firstname=u'Christophe') + req.create_entity('CWUser', + login=u'cdevienne', + upassword=u'cdevienne', + surname=u'de Vienne', + firstname=u'Christophe') e = req.execute('Any X WHERE X login "cdevienne"').one() self.assertEqual(e.surname, u'de Vienne') @@ -440,9 +444,81 @@ with self.assertRaises(MultipleResultsError): req.execute('Any X WHERE X is CWUser').one() + def test_first(self): + with self.admin_access.web_request() as req: + req.create_entity('CWUser', + login=u'cdevienne', + upassword=u'cdevienne', + surname=u'de Vienne', + firstname=u'Christophe') + e = req.execute('Any X WHERE X login "cdevienne"').first() + self.assertEqual(e.surname, u'de Vienne') + + e = req.execute( + 'Any X, N WHERE X login "cdevienne", X surname N').first() + self.assertEqual(e.surname, u'de Vienne') + + e = req.execute( + 'Any N, X WHERE X login "cdevienne", X surname N').first(col=1) + self.assertEqual(e.surname, u'de Vienne') + + def test_first_no_rows(self): + with self.admin_access.web_request() as req: + with self.assertRaises(NoResultError): + req.execute('Any X WHERE X login "patanok"').first() + + def test_first_multiple_rows(self): + with self.admin_access.web_request() as req: + req.create_entity( + 'CWUser', login=u'user1', upassword=u'cdevienne', + surname=u'de Vienne', firstname=u'Christophe') + req.create_entity( + 'CWUser', login=u'user2', upassword='adim', + surname=u'di mascio', firstname=u'adrien') + + e = req.execute('Any X ORDERBY X WHERE X is CWUser, ' + 'X login LIKE "user%"').first() + self.assertEqual(e.login, 'user1') + + def test_last(self): + with self.admin_access.web_request() as req: + req.create_entity('CWUser', + login=u'cdevienne', + upassword=u'cdevienne', + surname=u'de Vienne', + firstname=u'Christophe') + e = req.execute('Any X WHERE X login "cdevienne"').last() + self.assertEqual(e.surname, u'de Vienne') + + e = req.execute( + 'Any X, N WHERE X login "cdevienne", X surname N').last() + self.assertEqual(e.surname, u'de Vienne') + + e = req.execute( + 'Any N, X WHERE X login "cdevienne", X surname N').last(col=1) + self.assertEqual(e.surname, u'de Vienne') + + def test_last_no_rows(self): + with self.admin_access.web_request() as req: + with self.assertRaises(NoResultError): + req.execute('Any X WHERE X login "patanok"').last() + + def test_last_multiple_rows(self): + with self.admin_access.web_request() as req: + req.create_entity( + 'CWUser', login=u'user1', upassword=u'cdevienne', + surname=u'de Vienne', firstname=u'Christophe') + req.create_entity( + 'CWUser', login=u'user2', upassword='adim', + surname=u'di mascio', firstname=u'adrien') + + e = req.execute('Any X ORDERBY X WHERE X is CWUser, ' + 'X login LIKE "user%"').last() + self.assertEqual(e.login, 'user2') + def test_related_entity_optional(self): with self.admin_access.web_request() as req: - e = req.create_entity('Bookmark', title=u'aaaa', path=u'path') + req.create_entity('Bookmark', title=u'aaaa', path=u'path') rset = req.execute('Any B,U,L WHERE B bookmarked_by U?, U login L') entity, rtype = rset.related_entity(0, 2) self.assertEqual(entity, None) @@ -452,9 +528,9 @@ with self.admin_access.web_request() as req: e = req.create_entity('Bookmark', title=u'aaaa', path=u'path') rset = req.execute('Any X,N ORDERBY N WITH X,N BEING ' - '((Any X,N WHERE X is CWGroup, X name N)' - ' UNION ' - ' (Any X,N WHERE X is Bookmark, X title N))') + '((Any X,N WHERE X is CWGroup, X name N)' + ' UNION ' + ' (Any X,N WHERE X is Bookmark, X title N))') entity, rtype = rset.related_entity(0, 1) self.assertEqual(entity.eid, e.eid) self.assertEqual(rtype, 'title') @@ -468,9 +544,9 @@ with self.admin_access.web_request() as req: e = req.create_entity('Bookmark', title=u'aaaa', path=u'path') rset = req.execute('Any X,N ORDERBY N WHERE X is Bookmark WITH X,N BEING ' - '((Any X,N WHERE X is CWGroup, X name N)' - ' UNION ' - ' (Any X,N WHERE X is Bookmark, X title N))') + '((Any X,N WHERE X is CWGroup, X name N)' + ' UNION ' + ' (Any X,N WHERE X is Bookmark, X title N))') entity, rtype = rset.related_entity(0, 1) self.assertEqual(entity.eid, e.eid) self.assertEqual(rtype, 'title') @@ -480,9 +556,9 @@ with self.admin_access.web_request() as req: e = req.create_entity('Bookmark', title=u'aaaa', path=u'path') rset = req.execute('Any X,N ORDERBY N WITH N,X BEING ' - '((Any N,X WHERE X is CWGroup, X name N)' - ' UNION ' - ' (Any N,X WHERE X is Bookmark, X title N))') + '((Any N,X WHERE X is CWGroup, X name N)' + ' UNION ' + ' (Any N,X WHERE X is Bookmark, X title N))') entity, rtype = rset.related_entity(0, 1) self.assertEqual(entity.eid, e.eid) self.assertEqual(rtype, 'title') @@ -492,9 +568,9 @@ with self.admin_access.web_request() as req: e = req.create_entity('Bookmark', title=u'aaaa', path=u'path') rset = req.execute('Any X,X, N ORDERBY N WITH X,N BEING ' - '((Any X,N WHERE X is CWGroup, X name N)' - ' UNION ' - ' (Any X,N WHERE X is Bookmark, X title N))') + '((Any X,N WHERE X is CWGroup, X name N)' + ' UNION ' + ' (Any X,N WHERE X is Bookmark, X title N))') entity, rtype = rset.related_entity(0, 2) self.assertEqual(entity.eid, e.eid) self.assertEqual(rtype, 'title') @@ -505,14 +581,14 @@ req.create_entity('Bookmark', title=u'test bookmark', path=u'') req.execute('SET B bookmarked_by U WHERE U login "admin"') rset = req.execute('Any B,T,L WHERE B bookmarked_by U, U login L ' - 'WITH B,T BEING (Any B,T WHERE B is Bookmark, B title T)') + 'WITH B,T BEING (Any B,T WHERE B is Bookmark, B title T)') rset.related_entity(0, 2) def test_related_entity_subquery_outerjoin(self): with self.admin_access.web_request() as req: rset = req.execute('Any X,S,L WHERE X in_state S ' - 'WITH X, L BEING (Any X,MAX(L) GROUPBY X ' - 'WHERE X is CWUser, T? wf_info_for X, T creation_date L)') + 'WITH X, L BEING (Any X,MAX(L) GROUPBY X ' + 'WHERE X is CWUser, T? wf_info_for X, T creation_date L)') self.assertEqual(len(rset), 2) rset.related_entity(0, 1) rset.related_entity(0, 2) @@ -523,9 +599,9 @@ # make sure we have at least one element self.assertTrue(rset) self.assertEqual(set(e.e_schema.type for e in rset.entities(0)), - set(['CWUser',])) + set(['CWUser'])) self.assertEqual(set(e.e_schema.type for e in rset.entities(1)), - set(['CWGroup',])) + set(['CWGroup'])) def test_iter_rows_with_entities(self): with self.admin_access.web_request() as req: @@ -533,35 +609,35 @@ # make sure we have at least one element self.assertTrue(rset) out = list(rset.iter_rows_with_entities())[0] - self.assertEqual( out[0].login, out[1] ) - self.assertEqual( out[2].name, out[3] ) + self.assertEqual(out[0].login, out[1]) + self.assertEqual(out[2].name, out[3]) def test_printable_rql(self): with self.admin_access.web_request() as req: rset = req.execute(u'CWEType X WHERE X final FALSE') self.assertEqual(rset.printable_rql(), - 'Any X WHERE X final FALSE, X is CWEType') + 'Any X WHERE X final FALSE, X is CWEType') def test_searched_text(self): with self.admin_access.web_request() as req: rset = req.execute(u'Any X WHERE X has_text "foobar"') self.assertEqual(rset.searched_text(), 'foobar') - rset = req.execute(u'Any X WHERE X has_text %(text)s', {'text' : 'foo'}) + rset = req.execute(u'Any X WHERE X has_text %(text)s', {'text': 'foo'}) self.assertEqual(rset.searched_text(), 'foo') def test_union_limited_rql(self): with self.admin_access.web_request() as req: rset = req.execute('(Any X,N WHERE X is Bookmark, X title N)' - ' UNION ' - '(Any X,N WHERE X is CWGroup, X name N)') + ' UNION ' + '(Any X,N WHERE X is CWGroup, X name N)') rset.limit(2, 10, inplace=True) self.assertEqual(rset.limited_rql(), - 'Any A,B LIMIT 2 OFFSET 10 ' - 'WITH A,B BEING (' - '(Any X,N WHERE X is Bookmark, X title N) ' - 'UNION ' - '(Any X,N WHERE X is CWGroup, X name N)' - ')') + 'Any A,B LIMIT 2 OFFSET 10 ' + 'WITH A,B BEING (' + '(Any X,N WHERE X is Bookmark, X title N) ' + 'UNION ' + '(Any X,N WHERE X is CWGroup, X name N)' + ')') def test_possible_actions_cache(self): with self.admin_access.web_request() as req: @@ -572,8 +648,9 @@ def test_count_users_by_date(self): with self.admin_access.web_request() as req: - rset = req.execute('Any D, COUNT(U) GROUPBY D WHERE U is CWUser, U creation_date D') - self.assertEqual(rset.related_entity(0,0), (None, None)) + rset = req.execute('Any D, COUNT(U) GROUPBY D ' + 'WHERE U is CWUser, U creation_date D') + self.assertEqual(rset.related_entity(0, 0), (None, None)) def test_str(self): with self.admin_access.web_request() as req: @@ -594,15 +671,15 @@ def test_slice(self): rs = ResultSet([[12000, 'adim', u'Adim chez les pinguins'], [12000, 'adim', u'Jardiner facile'], - [13000, 'syt', u'Le carrelage en 42 leçons'], + [13000, 'syt', u'Le carrelage en 42 leçons'], [14000, 'nico', u'La tarte tatin en 15 minutes'], [14000, 'nico', u"L'épluchage du castor commun"]], - 'Any U, L, T WHERE U is CWUser, U login L,'\ - 'D created_by U, D title T', + ('Any U, L, T WHERE U is CWUser, U login L,' + 'D created_by U, D title T'), description=[['CWUser', 'String', 'String']] * 5) self.assertEqual(rs[1::2], - [[12000, 'adim', u'Jardiner facile'], - [14000, 'nico', u'La tarte tatin en 15 minutes']]) + [[12000, 'adim', u'Jardiner facile'], + [14000, 'nico', u'La tarte tatin en 15 minutes']]) def test_nonregr_symmetric_relation(self): # see https://www.cubicweb.org/ticket/4739253 @@ -611,7 +688,8 @@ cnx.create_entity('Personne', nom=u'denis', connait=p1) cnx.commit() rset = cnx.execute('Any X,Y WHERE X connait Y') - rset.get_entity(0, 1) # used to raise KeyError + rset.get_entity(0, 1) # used to raise KeyError + if __name__ == '__main__': unittest_main()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cubicweb/test/unittest_statsd.py Tue Jun 19 09:13:40 2018 +0200 @@ -0,0 +1,136 @@ +# -*- coding: utf-8 -*- +# copyright 2018 LOGILAB S.A. (Paris, FRANCE), all rights reserved. +# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr +# +# This file is part of CubicWeb. +# +# CubicWeb is free software: you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation, either version 2.1 of the License, or (at your option) +# any later version. +# +# CubicWeb is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License along +# with CubicWeb. If not, see <http://www.gnu.org/licenses/>. +"""unit tests for module cubicweb.statsd_logger""" + +import threading +import socket +import time +import re + +from unittest import TestCase +from cubicweb import statsd_logger as statsd + + +UDP_PORT = None +RUNNING = True +SOCK = socket.socket(socket.AF_INET, + socket.SOCK_DGRAM) +SOCK.settimeout(0.1) +STATSD = None +DATA = [] + + +def statsd_rcv(): + while RUNNING: + try: + data, addr = SOCK.recvfrom(1024) + if data: + rcv = [row.strip().decode() for row in data.splitlines()] + DATA.extend(rcv) + except socket.timeout: + pass + + +def setUpModule(*args): + global UDP_PORT, STATSD + SOCK.bind(('127.0.0.1', 0)) + UDP_PORT = SOCK.getsockname()[1] + STATSD = threading.Thread(target=statsd_rcv) + STATSD.start() + statsd.setup('test', ('127.0.0.1', UDP_PORT)) + + +def tearDownModule(*args): + global RUNNING + RUNNING = False + STATSD.join() + statsd.teardown() + + +class StatsdTC(TestCase): + + def setUp(self): + super(StatsdTC, self).setUp() + DATA[:] = [] + + def check_received(self, value): + for i in range(10): + if value in DATA: + break + time.sleep(0.01) + else: + self.assertIn(value, DATA) + + def check_received_ms(self, value): + value = re.compile(value.replace('?', '\d')) + for i in range(10): + if [x for x in DATA if value.match(x)]: + break + time.sleep(0.01) + else: + self.assertTrue([x for x in DATA if value.match(x)], DATA) + + def test_statsd_c(self): + statsd.statsd_c('context') + self.check_received('test.context:1|c') + statsd.statsd_c('context', 10) + self.check_received('test.context:10|c') + + def test_statsd_g(self): + statsd.statsd_g('context', 42) + self.check_received('test.context:42|g') + statsd.statsd_g('context', 'Igorrr') + self.check_received('test.context:Igorrr|g') + + def test_statsd_t(self): + statsd.statsd_t('context', 1) + self.check_received('test.context:1.0000|ms') + statsd.statsd_t('context', 10) + self.check_received('test.context:10.0000|ms') + statsd.statsd_t('context', 0.12344) + self.check_received('test.context:0.1234|ms') + statsd.statsd_t('context', 0.12345) + self.check_received('test.context:0.1235|ms') + + def test_decorator(self): + + @statsd.statsd_timeit + def measure_me_please(): + "some nice function" + return 42 + + self.assertEqual(measure_me_please.__doc__, + "some nice function") + + measure_me_please() + self.check_received_ms('test.measure_me_please:0.0???|ms') + self.check_received('test.measure_me_please:1|c') + + def test_context_manager(self): + + with statsd.statsd_timethis('cm'): + time.sleep(0.1) + + self.check_received_ms('test.cm:100.????|ms') + self.check_received('test.cm:1|c') + + +if __name__ == '__main__': + from unittest import main + main()
--- a/cubicweb/test/unittest_utils.py Mon Jun 18 10:04:08 2018 +0200 +++ b/cubicweb/test/unittest_utils.py Tue Jun 19 09:13:40 2018 +0200 @@ -109,6 +109,95 @@ 'itemcount': 10, 'permanentcount': 5}) + def test_clear_on_overflow(self): + """Tests that only non-permanent items in the cache are wiped-out on ceiling overflow + """ + c = QueryCache(ceiling=10) + # set 10 values + for x in range(10): + c[x] = x + # arrange for the first 5 to be permanent + for x in range(5): + for r in range(QueryCache._maxlevel + 2): + v = c[x] + self.assertEqual(v, x) + # Add the 11-th + c[10] = 10 + self.assertEqual(c._usage_report(), + {'transientcount': 0, + 'itemcount': 6, + 'permanentcount': 5}) + + def test_get_with_default(self): + """ + Tests the capability of QueryCache for retrieving items with a default value + """ + c = QueryCache(ceiling=20) + # set 10 values + for x in range(10): + c[x] = x + # arrange for the first 5 to be permanent + for x in range(5): + for r in range(QueryCache._maxlevel + 2): + v = c[x] + self.assertEqual(v, x) + self.assertEqual(c._usage_report(), + {'transientcount': 0, + 'itemcount': 10, + 'permanentcount': 5}) + # Test defaults for existing (including in permanents) + for x in range(10): + v = c.get(x, -1) + self.assertEqual(v, x) + # Test defaults for others + for x in range(10, 15): + v = c.get(x, -1) + self.assertEqual(v, -1) + + def test_iterkeys(self): + """ + Tests the iterating on keys in the cache + """ + c = QueryCache(ceiling=20) + # set 10 values + for x in range(10): + c[x] = x + # arrange for the first 5 to be permanent + for x in range(5): + for r in range(QueryCache._maxlevel + 2): + v = c[x] + self.assertEqual(v, x) + self.assertEqual(c._usage_report(), + {'transientcount': 0, + 'itemcount': 10, + 'permanentcount': 5}) + keys = sorted(c) + for x in range(10): + self.assertEquals(x, keys[x]) + + def test_items(self): + """ + Tests the iterating on key-value couples in the cache + """ + c = QueryCache(ceiling=20) + # set 10 values + for x in range(10): + c[x] = x + # arrange for the first 5 to be permanent + for x in range(5): + for r in range(QueryCache._maxlevel + 2): + v = c[x] + self.assertEqual(v, x) + self.assertEqual(c._usage_report(), + {'transientcount': 0, + 'itemcount': 10, + 'permanentcount': 5}) + content = sorted(c.items()) + for x in range(10): + self.assertEquals(x, content[x][0]) + self.assertEquals(x, content[x][1]) + + class UStringIOTC(TestCase): def test_boolean_value(self): self.assertTrue(UStringIO())
--- a/cubicweb/toolsutils.py Mon Jun 18 10:04:08 2018 +0200 +++ b/cubicweb/toolsutils.py Tue Jun 19 09:13:40 2018 +0200 @@ -19,19 +19,19 @@ from __future__ import print_function - # XXX move most of this in logilab.common (shellutils ?) import io -import os, sys +import os +import sys import subprocess -from os import listdir, makedirs, environ, chmod, walk, remove -from os.path import exists, join, abspath, normpath +from os import listdir, makedirs, chmod, walk, remove +from os.path import exists, join, normpath import re from rlcompleter import Completer try: import readline -except ImportError: # readline not available, no completion +except ImportError: # readline not available, no completion pass try: from os import symlink @@ -44,11 +44,13 @@ from logilab.common.clcommands import Command as BaseCommand from logilab.common.shellutils import ASK -from cubicweb import warning # pylint: disable=E0611 +from cubicweb import warning # pylint: disable=E0611 from cubicweb import ConfigurationError, ExecutionError + def underline_title(title, car='-'): - return title+'\n'+(car*len(title)) + return title + '\n' + (car * len(title)) + def iter_dir(directory, condition_file=None, ignore=()): """iterate on a directory""" @@ -56,12 +58,13 @@ if sub in ('CVS', '.svn', '.hg'): continue if condition_file is not None and \ - not exists(join(directory, sub, condition_file)): + not exists(join(directory, sub, condition_file)): continue if sub in ignore: continue yield sub + def create_dir(directory): """create a directory if it doesn't exist yet""" try: @@ -73,6 +76,7 @@ raise print('-> no need to create existing directory %s' % directory) + def create_symlink(source, target): """create a symbolic link""" if exists(target): @@ -80,16 +84,19 @@ symlink(source, target) print('[symlink] %s <-- %s' % (target, source)) + def create_copy(source, target): import shutil print('[copy] %s <-- %s' % (target, source)) shutil.copy2(source, target) + def rm(whatever): import shutil shutil.rmtree(whatever) print('-> removed %s' % whatever) + def show_diffs(appl_file, ref_file, askconfirm=True): """interactivly replace the old file with the new file according to user decision @@ -122,7 +129,10 @@ else: print('no diff between %s and %s' % (appl_file, ref_file)) + SKEL_EXCLUDE = ('*.py[co]', '*.orig', '*~', '*_flymake.py') + + def copy_skeleton(skeldir, targetdir, context, exclude=SKEL_EXCLUDE, askconfirm=False): import shutil @@ -148,7 +158,7 @@ if fname.endswith('.tmpl'): tfpath = tfpath[:-5] if not askconfirm or not exists(tfpath) or \ - ASK.confirm('%s exists, overwrite?' % tfpath): + ASK.confirm('%s exists, overwrite?' % tfpath): fill_templated_file(fpath, tfpath, context) print('[generate] %s <-- %s' % (tfpath, fpath)) elif exists(tfpath): @@ -157,12 +167,14 @@ shutil.copyfile(fpath, tfpath) shutil.copymode(fpath, tfpath) + def fill_templated_file(fpath, tfpath, context): with io.open(fpath, encoding='ascii') as fobj: template = fobj.read() with io.open(tfpath, 'w', encoding='ascii') as fobj: fobj.write(template % context) + def restrict_perms_to_user(filepath, log=None): """set -rw------- permission on the given file""" if log: @@ -198,7 +210,7 @@ # start a section section = option[1:-1] assert section not in config, \ - 'Section %s is defined more than once' % section + 'Section %s is defined more than once' % section config[section] = current = {} continue sys.stderr.write('ignoring malformed line\n%r\n' % line) @@ -218,6 +230,7 @@ _HDLRS = {} + class metacmdhandler(type): def __new__(mcs, name, bases, classdict): cls = super(metacmdhandler, mcs).__new__(mcs, name, bases, classdict) @@ -229,6 +242,7 @@ @add_metaclass(metacmdhandler) class CommandHandler(object): """configuration specific helper for cubicweb-ctl commands""" + def __init__(self, config): self.config = config @@ -258,24 +272,25 @@ CONNECT_OPTIONS = ( ("user", - {'short': 'u', 'type' : 'string', 'metavar': '<user>', + {'short': 'u', 'type': 'string', 'metavar': '<user>', 'help': 'connect as <user> instead of being prompted to give it.', } ), ("password", - {'short': 'p', 'type' : 'password', 'metavar': '<password>', + {'short': 'p', 'type': 'password', 'metavar': '<password>', 'help': 'automatically give <password> for authentication instead of \ being prompted to give it.', }), ("host", - {'short': 'H', 'type' : 'string', 'metavar': '<hostname>', + {'short': 'H', 'type': 'string', 'metavar': '<hostname>', 'default': None, 'help': 'specify the name server\'s host name. Will be detected by \ broadcast if not provided.', }), - ) +) -## cwshell helpers ############################################################# +# cwshell helpers ############################################################# + class AbstractMatcher(object): """Abstract class for CWShellCompleter's matchers. @@ -350,7 +365,7 @@ 'rql_offset': len(func_prefix) + 2, # incomplete rql query 'rql_query': parameters_text, - } + } def possible_matches(self, text): """call ``rql.suggestions`` component to complete user's input. @@ -372,6 +387,7 @@ class DefaultMatcher(AbstractMatcher): """Default matcher: delegate to standard's `rlcompleter.Completer`` class """ + def __init__(self, local_ctx): self.completer = Completer(local_ctx) @@ -421,7 +437,7 @@ self.matches = matches break else: - return None # no matcher able to handle `text` + return None # no matcher able to handle `text` try: return self.matches[state] except IndexError:
--- a/cubicweb/utils.py Mon Jun 18 10:04:08 2018 +0200 +++ b/cubicweb/utils.py Tue Jun 19 09:13:40 2018 +0200 @@ -632,6 +632,29 @@ with self._lock: return len(self._data) + def items(self): + """Get an iterator over the dictionary's items: (key, value) pairs""" + with self._lock: + for k, v in self._data.items(): + yield k, v + + def get(self, k, default=None): + """Get the value associated to the specified key + + :param k: The key to look for + :param default: The default value when the key is not found + :return: The associated value (or the default value) + """ + try: + return self._data[k] + except KeyError: + return default + + def __iter__(self): + with self._lock: + for k in iter(self._data): + yield k + def __getitem__(self, k): with self._lock: if k in self._permanent: @@ -689,10 +712,20 @@ break level = v else: - # we removed cruft but everything is permanent + # we removed cruft if len(self._data) >= self._max: - logger.warning('Cache %s is full.' % id(self)) - self._clear() + if len(self._permanent) >= self._max: + # we really are full with permanents => clear + logger.warning('Cache %s is full.' % id(self)) + self._clear() + else: + # pathological case where _transient was probably empty ... + # drop all non-permanents + to_drop = set(self._data.keys()).difference(self._permanent) + for k in to_drop: + # should not be in _transient + assert k not in self._transient + self._data.pop(k, None) def _usage_report(self): with self._lock:
--- a/cubicweb/web/data/cubicweb.widgets.js Mon Jun 18 10:04:08 2018 +0200 +++ b/cubicweb/web/data/cubicweb.widgets.js Tue Jun 19 09:13:40 2018 +0200 @@ -151,7 +151,7 @@ */ var settings = $(this.element).data('settings'); var value = this.valueMethod.apply( this.element, arguments ); - if (settings.multiple & arguments.length === 0) { + if (settings.multiple && arguments.length === 0) { return extractLast(value); } return value
--- a/cubicweb/web/test/unittest_webconfig.py Mon Jun 18 10:04:08 2018 +0200 +++ b/cubicweb/web/test/unittest_webconfig.py Tue Jun 19 09:13:40 2018 +0200 @@ -45,7 +45,7 @@ def test_locate_resource(self): self.assertIn('FILE_ICON', self.config.uiprops) rname = self.config.uiprops['FILE_ICON'].replace(self.config.datadir_url, '') - self.assertIn('file', self.config.locate_resource(rname)[0].split(os.sep)) + self.assertIn('cubicweb_file', self.config.locate_resource(rname)[0].split(os.sep)) cubicwebcsspath = self.config.locate_resource('cubicweb.css')[0].split(os.sep) # 'shared' if tests under apycot @@ -57,7 +57,7 @@ wdocfiles = list(self.config.locate_all_files('toc.xml')) for fpath in wdocfiles: self.assertTrue(path.exists(fpath), fpath) - for expected in [path.join('cubes', 'file', 'wdoc', 'toc.xml'), + for expected in [path.join('cubicweb_file', 'wdoc', 'toc.xml'), path.join('cubicweb', 'web', 'wdoc', 'toc.xml')]: for fpath in wdocfiles: if fpath.endswith(expected):
--- a/cubicweb/web/views/uicfg.py Mon Jun 18 10:04:08 2018 +0200 +++ b/cubicweb/web/views/uicfg.py Tue Jun 19 09:13:40 2018 +0200 @@ -233,7 +233,7 @@ for key in list(self._tagdefs): stype, rtype, otype, role = key rschema = schema.rschema(rtype) - if stype == '*' and stype == '*': + if stype == '*' and otype == '*': concrete_rdefs = rschema.rdefs.keys() elif stype == '*': concrete_rdefs = zip(rschema.subjects(otype), repeat(otype))
--- a/flake8-ok-files.txt Mon Jun 18 10:04:08 2018 +0200 +++ b/flake8-ok-files.txt Tue Jun 19 09:13:40 2018 +0200 @@ -60,6 +60,7 @@ cubicweb/server/test/data-schema2sql/__init__.py cubicweb/server/test/unittest_checkintegrity.py cubicweb/server/test/unittest_datafeed.py +cubicweb/server/test/unittest_edition.py cubicweb/server/test/unittest_ldapsource.py cubicweb/server/test/unittest_migractions.py cubicweb/server/test/unittest_serverctl.py @@ -102,10 +103,13 @@ cubicweb/test/unittest_repoapi.py cubicweb/test/unittest_req.py cubicweb/test/unittest_rqlrewrite.py +cubicweb/test/unittest_rset.py cubicweb/test/unittest_rtags.py cubicweb/test/unittest_schema.py +cubicweb/test/unittest_statsd.py cubicweb/test/unittest_toolsutils.py cubicweb/test/unittest_wfutils.py +cubicweb/toolsutils.py cubicweb/web/application.py cubicweb/web/formwidgets.py cubicweb/web/test/data/entities.py