# HG changeset patch # User Sylvain Thénault # Date 1269416421 -3600 # Node ID 5594aadb740ef4494ff84f3825028a5bd7635dc9 # Parent 1f3d8946ea8465810e6420b4c4871762b34cfa13# Parent 706a5931e7655f8a2898d8ea24a52695d766a0c8 merge diff -r 706a5931e765 -r 5594aadb740e cwvreg.py --- a/cwvreg.py Wed Mar 24 07:55:31 2010 +0100 +++ b/cwvreg.py Wed Mar 24 08:40:21 2010 +0100 @@ -312,9 +312,7 @@ """set instance'schema and load application objects""" self._set_schema(schema) # now we can load application's web objects - searchpath = self.config.vregistry_path() - self.reset(searchpath, force_reload=False) - self.register_objects(searchpath, force_reload=False) + self._reload(self.config.vregistry_path(), force_reload=False) # map lowered entity type names to their actual name self.case_insensitive_etypes = {} for eschema in self.schema.entities(): @@ -323,6 +321,14 @@ clear_cache(eschema, 'ordered_relations') clear_cache(eschema, 'meta_attributes') + def _reload(self, path, force_reload): + CW_EVENT_MANAGER.emit('before-registry-reload') + # modification detected, reset and reload + self.reset(path, force_reload) + super(CubicWebVRegistry, self).register_objects( + path, force_reload, self.config.extrapath) + CW_EVENT_MANAGER.emit('after-registry-reload') + def _set_schema(self, schema): """set instance'schema""" self.schema = schema @@ -363,12 +369,7 @@ super(CubicWebVRegistry, self).register_objects( path, force_reload, self.config.extrapath) except RegistryOutOfDate: - CW_EVENT_MANAGER.emit('before-registry-reload') - # modification detected, reset and reload - self.reset(path, force_reload) - super(CubicWebVRegistry, self).register_objects( - path, force_reload, self.config.extrapath) - CW_EVENT_MANAGER.emit('after-registry-reload') + self._reload(path, force_reload) def initialization_completed(self): """cw specific code once vreg initialization is completed: diff -r 706a5931e765 -r 5594aadb740e debian/control --- a/debian/control Wed Mar 24 07:55:31 2010 +0100 +++ b/debian/control Wed Mar 24 08:40:21 2010 +0100 @@ -98,7 +98,7 @@ Package: cubicweb-common Architecture: all XB-Python-Version: ${python:Versions} -Depends: ${python:Depends}, graphviz, gettext, python-logilab-mtconverter (>= 0.6.0), python-logilab-common (>= 0.49.0), python-yams (>= 0.28.0), python-rql (>= 0.25.0), python-lxml +Depends: ${python:Depends}, graphviz, gettext, python-logilab-mtconverter (>= 0.6.0), python-logilab-common (>= 0.49.0), python-yams (>= 0.28.1), python-rql (>= 0.25.0), python-lxml Recommends: python-simpletal (>= 4.0), python-crypto Conflicts: cubicweb-core Replaces: cubicweb-core diff -r 706a5931e765 -r 5594aadb740e entity.py --- a/entity.py Wed Mar 24 07:55:31 2010 +0100 +++ b/entity.py Wed Mar 24 08:40:21 2010 +0100 @@ -60,7 +60,7 @@ :cvar skip_copy_for: a list of relations that should be skipped when copying this kind of entity. Note that some relations such as composite relations or relations that have '?1' as object - cardinality are always skipped. + cardinality are always skipped. """ __registry__ = 'etypes' __select__ = yes() @@ -225,6 +225,47 @@ def __cmp__(self, other): raise NotImplementedError('comparison not implemented for %s' % self.__class__) + def __getitem__(self, key): + if key == 'eid': + warn('[3.7] entity["eid"] is deprecated, use entity.eid instead', + DeprecationWarning, stacklevel=2) + return self.eid + return super(Entity, self).__getitem__(key) + + def __setitem__(self, attr, value): + """override __setitem__ to update self.edited_attributes. + + Typically, a before_update_hook could do:: + + entity['generated_attr'] = generated_value + + and this way, edited_attributes will be updated accordingly + """ + if attr == 'eid': + warn('[3.7] entity["eid"] = value is deprecated, use entity.eid = value instead', + DeprecationWarning, stacklevel=2) + self.eid = value + else: + super(Entity, self).__setitem__(attr, value) + if hasattr(self, 'edited_attributes'): + self.edited_attributes.add(attr) + self.skip_security_attributes.add(attr) + + def setdefault(self, attr, default): + """override setdefault to update self.edited_attributes""" + super(Entity, self).setdefault(attr, default) + if hasattr(self, 'edited_attributes'): + self.edited_attributes.add(attr) + self.skip_security_attributes.add(attr) + + def rql_set_value(self, attr, value): + """call by rql execution plan when some attribute is modified + + don't use dict api in such case since we don't want attribute to be + added to skip_security_attributes. + """ + super(Entity, self).__setitem__(attr, value) + def pre_add_hook(self): """hook called by the repository before doing anything to add the entity (before_add entity hooks have not been called yet). This give the @@ -235,7 +276,7 @@ return self def set_eid(self, eid): - self.eid = self['eid'] = eid + self.eid = eid def has_eid(self): """return True if the entity has an attributed eid (False @@ -781,10 +822,6 @@ haseid = 'eid' in self self._cw_completed = False self.clear() - # set eid if it was in, else we may get nasty error while editing this - # entity if it's bound to a repo session - if haseid: - self['eid'] = self.eid # clear relations cache for rschema, _, role in self.e_schema.relation_definitions(): self.clear_related_cache(rschema.type, role) @@ -840,13 +877,19 @@ # server side utilities ################################################### + @property + def skip_security_attributes(self): + try: + return self._skip_security_attributes + except: + self._skip_security_attributes = set() + return self._skip_security_attributes + def set_defaults(self): """set default values according to the schema""" - self._default_set = set() for attr, value in self.e_schema.defaults(): if not self.has_key(attr): self[str(attr)] = value - self._default_set.add(attr) def check(self, creation=False): """check this entity against its schema. Only final relation @@ -858,7 +901,15 @@ _ = unicode else: _ = self._cw._ - self.e_schema.check(self, creation=creation, _=_) + if creation or not hasattr(self, 'edited_attributes'): + # on creations, we want to check all relations, especially + # required attributes + relations = None + else: + relations = [self._cw.vreg.schema.rschema(rtype) + for rtype in self.edited_attributes] + self.e_schema.check(self, creation=creation, _=_, + relations=relations) def fti_containers(self, _done=None): if _done is None: @@ -932,8 +983,6 @@ def __set__(self, eobj, value): eobj[self._attrname] = value - if hasattr(eobj, 'edited_attributes'): - eobj.edited_attributes.add(self._attrname) class Relation(object): """descriptor that controls schema relation access""" diff -r 706a5931e765 -r 5594aadb740e hooks/security.py --- a/hooks/security.py Wed Mar 24 07:55:31 2010 +0100 +++ b/hooks/security.py Wed Mar 24 08:40:21 2010 +0100 @@ -16,17 +16,20 @@ def check_entity_attributes(session, entity, editedattrs=None): eid = entity.eid eschema = entity.e_schema - # ._default_set is only there on entity creation to indicate unspecified - # attributes which has been set to a default value defined in the schema - defaults = getattr(entity, '_default_set', ()) + # .skip_security_attributes is there to bypass security for attributes + # set by hooks by modifying the entity's dictionnary + dontcheck = entity.skip_security_attributes if editedattrs is None: try: editedattrs = entity.edited_attributes except AttributeError: - editedattrs = entity + editedattrs = entity # XXX unexpected for attr in editedattrs: - if attr in defaults: + try: + dontcheck.remove(attr) continue + except KeyError: + pass rdef = eschema.rdef(attr) if rdef.final: # non final relation are checked by other hooks # add/delete should be equivalent (XXX: unify them into 'update' ?) diff -r 706a5931e765 -r 5594aadb740e hooks/storages.py --- a/hooks/storages.py Wed Mar 24 07:55:31 2010 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,41 +0,0 @@ -"""hooks to handle attributes mapped to a custom storage -""" -from cubicweb.server.hook import Hook -from cubicweb.server.sources.storages import ETYPE_ATTR_STORAGE - - -class BFSSHook(Hook): - """abstract class for bytes file-system storage hooks""" - __abstract__ = True - category = 'bfss' - - -class PreAddEntityHook(BFSSHook): - """""" - __regid__ = 'bfss_add_entity' - events = ('before_add_entity', ) - - def __call__(self): - etype = self.entity.__regid__ - for attr in ETYPE_ATTR_STORAGE.get(etype, ()): - ETYPE_ATTR_STORAGE[etype][attr].entity_added(self.entity, attr) - -class PreUpdateEntityHook(BFSSHook): - """""" - __regid__ = 'bfss_update_entity' - events = ('before_update_entity', ) - - def __call__(self): - etype = self.entity.__regid__ - for attr in ETYPE_ATTR_STORAGE.get(etype, ()): - ETYPE_ATTR_STORAGE[etype][attr].entity_updated(self.entity, attr) - -class PreDeleteEntityHook(BFSSHook): - """""" - __regid__ = 'bfss_delete_entity' - events = ('before_delete_entity', ) - - def __call__(self): - etype = self.entity.__regid__ - for attr in ETYPE_ATTR_STORAGE.get(etype, ()): - ETYPE_ATTR_STORAGE[etype][attr].entity_deleted(self.entity, attr) diff -r 706a5931e765 -r 5594aadb740e server/sources/native.py --- a/server/sources/native.py Wed Mar 24 07:55:31 2010 +0100 +++ b/server/sources/native.py Wed Mar 24 08:40:21 2010 +0100 @@ -19,6 +19,7 @@ from threading import Lock from datetime import datetime from base64 import b64decode, b64encode +from contextlib import contextmanager from logilab.common.compat import any from logilab.common.cache import Cache @@ -191,6 +192,8 @@ self._cache = Cache(repo.config['rql-cache-size']) self._temp_table_data = {} self._eid_creation_lock = Lock() + # (etype, attr) / storage mapping + self._storages = {} # XXX no_sqlite_wrap trick since we've a sqlite locking pb when # running unittest_multisources with the wrapping below if self.dbdriver == 'sqlite' and \ @@ -267,6 +270,18 @@ def unmap_attribute(self, etype, attr): self._rql_sqlgen.attr_map.pop('%s.%s' % (etype, attr), None) + def set_storage(self, etype, attr, storage): + storage_dict = self._storages.setdefault(etype, {}) + storage_dict[attr] = storage + self.map_attribute(etype, attr, storage.sqlgen_callback) + + def unset_storage(self, etype, attr): + self._storages[etype].pop(attr) + # if etype has no storage left, remove the entry + if not self._storages[etype]: + del self._storages[etype] + self.unmap_attribute(etype, attr) + # ISource interface ####################################################### def compile_rql(self, rql, sols): @@ -402,40 +417,63 @@ except KeyError: continue + @contextmanager + def _storage_handler(self, entity, event): + # 1/ memorize values as they are before the storage is called. + # For instance, the BFSStorage will replace the `data` + # binary value with a Binary containing the destination path + # on the filesystem. To make the entity.data usage absolutely + # transparent, we'll have to reset entity.data to its binary + # value once the SQL query will be executed + orig_values = {} + etype = entity.__regid__ + for attr, storage in self._storages.get(etype, {}).items(): + if attr in entity.edited_attributes: + orig_values[attr] = entity[attr] + handler = getattr(storage, 'entity_%s' % event) + handler(entity, attr) + yield # 2/ execute the source's instructions + # 3/ restore original values + for attr, value in orig_values.items(): + entity[attr] = value + def add_entity(self, session, entity): """add a new entity to the source""" - attrs = self.preprocess_entity(entity) - sql = self.sqlgen.insert(SQL_PREFIX + entity.__regid__, attrs) - self.doexec(session, sql, attrs) - if session.undoable_action('C', entity.__regid__): - self._record_tx_action(session, 'tx_entity_actions', 'C', - etype=entity.__regid__, eid=entity.eid) + with self._storage_handler(entity, 'added'): + attrs = self.preprocess_entity(entity) + sql = self.sqlgen.insert(SQL_PREFIX + entity.__regid__, attrs) + self.doexec(session, sql, attrs) + if session.undoable_action('C', entity.__regid__): + self._record_tx_action(session, 'tx_entity_actions', 'C', + etype=entity.__regid__, eid=entity.eid) def update_entity(self, session, entity): """replace an entity in the source""" - attrs = self.preprocess_entity(entity) - if session.undoable_action('U', entity.__regid__): - changes = self._save_attrs(session, entity, attrs) - self._record_tx_action(session, 'tx_entity_actions', 'U', - etype=entity.__regid__, eid=entity.eid, - changes=self._binary(dumps(changes))) - sql = self.sqlgen.update(SQL_PREFIX + entity.__regid__, attrs, - ['cw_eid']) - self.doexec(session, sql, attrs) + with self._storage_handler(entity, 'updated'): + attrs = self.preprocess_entity(entity) + if session.undoable_action('U', entity.__regid__): + changes = self._save_attrs(session, entity, attrs) + self._record_tx_action(session, 'tx_entity_actions', 'U', + etype=entity.__regid__, eid=entity.eid, + changes=self._binary(dumps(changes))) + sql = self.sqlgen.update(SQL_PREFIX + entity.__regid__, attrs, + ['cw_eid']) + self.doexec(session, sql, attrs) def delete_entity(self, session, entity): """delete an entity from the source""" - if session.undoable_action('D', entity.__regid__): - attrs = [SQL_PREFIX + r.type - for r in entity.e_schema.subject_relations() - if (r.final or r.inlined) and not r in VIRTUAL_RTYPES] - changes = self._save_attrs(session, entity, attrs) - self._record_tx_action(session, 'tx_entity_actions', 'D', - etype=entity.__regid__, eid=entity.eid, - changes=self._binary(dumps(changes))) - attrs = {'cw_eid': entity.eid} - sql = self.sqlgen.delete(SQL_PREFIX + entity.__regid__, attrs) - self.doexec(session, sql, attrs) + with self._storage_handler(entity, 'deleted'): + if session.undoable_action('D', entity.__regid__): + attrs = [SQL_PREFIX + r.type + for r in entity.e_schema.subject_relations() + if (r.final or r.inlined) and not r in VIRTUAL_RTYPES] + changes = self._save_attrs(session, entity, attrs) + self._record_tx_action(session, 'tx_entity_actions', 'D', + etype=entity.__regid__, eid=entity.eid, + changes=self._binary(dumps(changes))) + attrs = {'cw_eid': entity.eid} + sql = self.sqlgen.delete(SQL_PREFIX + entity.__regid__, attrs) + self.doexec(session, sql, attrs) def _add_relation(self, session, subject, rtype, object, inlined=False): """add a relation to the source""" diff -r 706a5931e765 -r 5594aadb740e server/sources/storages.py --- a/server/sources/storages.py Wed Mar 24 07:55:31 2010 +0100 +++ b/server/sources/storages.py Wed Mar 24 08:40:21 2010 +0100 @@ -4,16 +4,11 @@ from cubicweb import Binary from cubicweb.server.hook import Operation - -ETYPE_ATTR_STORAGE = {} def set_attribute_storage(repo, etype, attr, storage): - ETYPE_ATTR_STORAGE.setdefault(etype, {})[attr] = storage - repo.system_source.map_attribute(etype, attr, storage.sqlgen_callback) + repo.system_source.set_storage(etype, attr, storage) def unset_attribute_storage(repo, etype, attr): - ETYPE_ATTR_STORAGE.setdefault(etype, {}).pop(attr, None) - repo.system_source.unmap_attribute(etype, attr) - + repo.system_source.unset_storage(etype, attr) class Storage(object): """abstract storage""" diff -r 706a5931e765 -r 5594aadb740e server/sqlutils.py --- a/server/sqlutils.py Wed Mar 24 07:55:31 2010 +0100 +++ b/server/sqlutils.py Wed Mar 24 08:40:21 2010 +0100 @@ -214,7 +214,8 @@ """ attrs = {} eschema = entity.e_schema - for attr, value in entity.items(): + for attr in entity.edited_attributes: + value = entity[attr] rschema = eschema.subjrels[attr] if rschema.final: atype = str(entity.e_schema.destination(attr)) @@ -236,6 +237,7 @@ elif isinstance(value, Binary): value = self._binary(value.getvalue()) attrs[SQL_PREFIX+str(attr)] = value + attrs[SQL_PREFIX+'eid'] = entity.eid return attrs diff -r 706a5931e765 -r 5594aadb740e server/ssplanner.py --- a/server/ssplanner.py Wed Mar 24 07:55:31 2010 +0100 +++ b/server/ssplanner.py Wed Mar 24 08:40:21 2010 +0100 @@ -473,7 +473,7 @@ value = row[index] index += 1 if rorder == InsertRelationsStep.FINAL: - edef[rtype] = value + edef.rql_set_value(rtype, value) elif rorder == InsertRelationsStep.RELATION: self.plan.add_relation_def( (edef, rtype, value) ) edef.querier_pending_relations[(rtype, 'subject')] = value @@ -564,7 +564,7 @@ edef = edefs[eid] except KeyError: edefs[eid] = edef = session.entity_from_eid(eid) - edef[str(rschema)] = rhsval + edef.rql_set_value(str(rschema), rhsval) else: repo.glob_add_relation(session, lhsval, str(rschema), rhsval) result[i] = newrow diff -r 706a5931e765 -r 5594aadb740e server/test/unittest_storage.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/server/test/unittest_storage.py Wed Mar 24 08:40:21 2010 +0100 @@ -0,0 +1,91 @@ +"""unit tests for module cubicweb.server.sources.storages + +:organization: Logilab +:copyright: 2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. +:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr +:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses +""" + +from logilab.common.testlib import unittest_main +from cubicweb.devtools.testlib import CubicWebTC + +import os.path as osp +import shutil +import tempfile + +from cubicweb import Binary +from cubicweb.selectors import implements +from cubicweb.server.sources import storages +from cubicweb.server.hook import Hook, Operation + +class DummyBeforeHook(Hook): + __regid__ = 'dummy-before-hook' + __select__ = Hook.__select__ & implements('File') + events = ('before_add_entity',) + + def __call__(self): + self._cw.transaction_data['orig_file_value'] = self.entity.data.getvalue() + + +class DummyAfterHook(Hook): + __regid__ = 'dummy-after-hook' + __select__ = Hook.__select__ & implements('File') + events = ('after_add_entity',) + + def __call__(self): + # new value of entity.data should be the same as before + oldvalue = self._cw.transaction_data['orig_file_value'] + assert oldvalue == self.entity.data.getvalue() + + +class StorageTC(CubicWebTC): + + def setup_database(self): + self.tempdir = tempfile.mkdtemp() + bfs_storage = storages.BytesFileSystemStorage(self.tempdir) + storages.set_attribute_storage(self.repo, 'File', 'data', bfs_storage) + + def tearDown(self): + super(CubicWebTC, self).tearDown() + storages.unset_attribute_storage(self.repo, 'File', 'data') + shutil.rmtree(self.tempdir) + + + def create_file(self, content): + req = self.request() + return req.create_entity('File', data=Binary(content), + data_format=u'text/plain', data_name=u'foo') + + def test_bfs_storage(self): + f1 = self.create_file(content='the-data') + expected_filepath = osp.join(self.tempdir, '%s_data' % f1.eid) + self.failUnless(osp.isfile(expected_filepath)) + self.assertEquals(file(expected_filepath).read(), 'the-data') + + def test_sqlite_fspath(self): + f1 = self.create_file(content='the-data') + expected_filepath = osp.join(self.tempdir, '%s_data' % f1.eid) + fspath = self.execute('Any fspath(F, "File", "data") WHERE F eid %(f)s', + {'f': f1.eid})[0][0] + self.assertEquals(fspath.getvalue(), expected_filepath) + + def test_fs_importing_doesnt_touch_path(self): + self.session.transaction_data['fs_importing'] = True + f1 = self.session.create_entity('File', data=Binary('/the/path'), + data_format=u'text/plain', data_name=u'foo') + fspath = self.execute('Any fspath(F, "File", "data") WHERE F eid %(f)s', + {'f': f1.eid})[0][0] + self.assertEquals(fspath.getvalue(), '/the/path') + + def test_storage_transparency(self): + self.vreg._loadedmods[__name__] = {} + self.vreg.register(DummyBeforeHook) + self.vreg.register(DummyAfterHook) + try: + self.create_file(content='the-data') + finally: + self.vreg.unregister(DummyBeforeHook) + self.vreg.unregister(DummyAfterHook) + +if __name__ == '__main__': + unittest_main()