# HG changeset patch # User Sylvain Thénault # Date 1308058629 -7200 # Node ID 2983dd24494addb320aa710e6b93765b97be7a24 # Parent cb0f4da64e8629dd07755b5231ec9960ddb60b3c [repository] refactor/cleanup entity deletion methods * kill unused Repository.delete_info_multi (no leading _) * kill unnecessary system_source.delete_info (almost no benefit over delete_info_multi) * stop giving entities list *and* extid list, gather them from the entity when needed * move pending eids handling into glob_delete_entities, so one can call it directly without worrying about that diff -r cb0f4da64e86 -r 2983dd24494a server/migractions.py --- a/server/migractions.py Tue Jun 14 13:46:36 2011 +0200 +++ b/server/migractions.py Tue Jun 14 15:37:09 2011 +0200 @@ -944,23 +944,19 @@ # triggered by schema synchronization hooks. session = self.session for rdeftype in ('CWRelation', 'CWAttribute'): - thispending = set() - for eid, in self.sqlexec('SELECT cw_eid FROM cw_%s ' - 'WHERE cw_from_entity=%%(eid)s OR ' - ' cw_to_entity=%%(eid)s' % rdeftype, - {'eid': oldeid}, ask_confirm=False): - # we should add deleted eids into pending eids else we may - # get some validation error on commit since integrity hooks - # may think some required relation is missing... This also ensure - # repository caches are properly cleanup - hook.CleanupDeletedEidsCacheOp.get_instance(session).add_data(eid) - # and don't forget to remove record from system tables - self.repo.system_source.delete_info( - session, session.entity_from_eid(eid, rdeftype), - 'system', None) - thispending.add(eid) - self.sqlexec('DELETE FROM cw_%s ' - 'WHERE cw_from_entity=%%(eid)s OR ' + thispending = set( (eid for eid, in self.sqlexec( + 'SELECT cw_eid FROM cw_%s WHERE cw_from_entity=%%(eid)s OR ' + ' cw_to_entity=%%(eid)s' % rdeftype, + {'eid': oldeid}, ask_confirm=False)) ) + # we should add deleted eids into pending eids else we may + # get some validation error on commit since integrity hooks + # may think some required relation is missing... This also ensure + # repository caches are properly cleanup + hook.CleanupDeletedEidsCacheOp.get_instance(session).union(thispending) + # and don't forget to remove record from system tables + entities = [session.entity_from_eid(eid, rdeftype) for eid in thispending] + self.repo.system_source.delete_info_multi(session, entities, 'system') + self.sqlexec('DELETE FROM cw_%s WHERE cw_from_entity=%%(eid)s OR ' 'cw_to_entity=%%(eid)s' % rdeftype, {'eid': oldeid}, ask_confirm=False) # now we have to manually cleanup relations pointing to deleted diff -r cb0f4da64e86 -r 2983dd24494a server/repository.py --- a/server/repository.py Tue Jun 14 13:46:36 2011 +0200 +++ b/server/repository.py Tue Jun 14 15:37:09 2011 +0200 @@ -1097,17 +1097,6 @@ hook.CleanupDeletedEidsCacheOp.get_instance(session).add_data(entity.eid) self._delete_info(session, entity, sourceuri, extid, scleanup) - def delete_info_multi(self, session, entities, sourceuri, extids, scleanup=None): - """same as delete_info but accepts a list of entities and - extids with the same etype and belonging to the same source - """ - # mark eid as being deleted in session info and setup cache update - # operation - op = hook.CleanupDeletedEidsCacheOp.get_instance(session) - for entity in entities: - op.add_data(entity.eid) - self._delete_info_multi(session, entities, sourceuri, extids, scleanup) - def _delete_info(self, session, entity, sourceuri, extid, scleanup=None): """delete system information on deletion of an entity: * delete all remaining relations from/to this entity @@ -1139,16 +1128,15 @@ except: self.exception('error while cascading delete for entity %s ' 'from %s. RQL: %s', entity, sourceuri, rql) - self.system_source.delete_info(session, entity, sourceuri, extid) + self.system_source.delete_info_multi(session, [entity], sourceuri) - def _delete_info_multi(self, session, entities, sourceuri, extids, scleanup=None): + def _delete_info_multi(self, session, entities, sourceuri, scleanup=None): """same as _delete_info but accepts a list of entities with the same etype and belinging to the same source. """ pendingrtypes = session.transaction_data.get('pendingrtypes', ()) # delete remaining relations: if user can delete the entity, he can # delete all its relations without security checking - assert entities and len(entities) == len(extids) with security_enabled(session, read=False, write=False): eids = [_e.eid for _e in entities] in_eids = ','.join((str(eid) for eid in eids)) @@ -1170,7 +1158,7 @@ except: self.exception('error while cascading delete for entity %s ' 'from %s. RQL: %s', entities, sourceuri, rql) - self.system_source.delete_info_multi(session, entities, sourceuri, extids) + self.system_source.delete_info_multi(session, entities, sourceuri) def locate_relation_source(self, session, subject, rtype, object): subjsource = self.source_from_eid(subject, session) @@ -1345,6 +1333,12 @@ def glob_delete_entities(self, session, eids): """delete a list of entities and all related entities from the repository""" + # mark eids as being deleted in session info and setup cache update + # operation (register pending eids before actual deletion to avoid + # multiple call to glob_delete_entities) + op = hook.CleanupDeletedEidsCacheOp.get_instance(session) + eids = eids - op._container + op._container |= eids data_by_etype_source = {} # values are ([list of eids], # [list of extid], # [list of entities]) @@ -1356,27 +1350,23 @@ for eid in eids: etype, sourceuri, extid = self.type_and_source_from_eid(eid, session) + # XXX should cache entity's cw_metainformation entity = session.entity_from_eid(eid, etype) - _key = (etype, sourceuri) - if _key not in data_by_etype_source: - data_by_etype_source[_key] = ([eid], [extid], [entity]) - else: - _data = data_by_etype_source[_key] - _data[0].append(eid) - _data[1].append(extid) - _data[2].append(entity) - for (etype, sourceuri), (eids, extids, entities) in data_by_etype_source.iteritems(): + try: + data_by_etype_source[(etype, sourceuri)].append(entity) + except KeyError: + data_by_etype_source[(etype, sourceuri)] = [entity] + for (etype, sourceuri), entities in data_by_etype_source.iteritems(): if server.DEBUG & server.DBG_REPO: - print 'DELETE entities', etype, eids - #print 'DELETE entities', etype, len(eids) + print 'DELETE entities', etype, [entity.eid for entity in entities] source = self.sources_by_uri[sourceuri] if source.should_call_hooks: self.hm.call_hooks('before_delete_entity', session, entities=entities) - self._delete_info_multi(session, entities, sourceuri, extids) # xxx + self._delete_info_multi(session, entities, sourceuri) source.delete_entities(session, entities) if source.should_call_hooks: self.hm.call_hooks('after_delete_entity', session, entities=entities) - # don't clear cache here this is done in a hook on commit + # don't clear cache here, it is done in a hook on commit def glob_add_relation(self, session, subject, rtype, object): """add a relation to the repository""" diff -r cb0f4da64e86 -r 2983dd24494a server/sources/__init__.py --- a/server/sources/__init__.py Tue Jun 14 13:46:36 2011 +0200 +++ b/server/sources/__init__.py Tue Jun 14 15:37:09 2011 +0200 @@ -462,19 +462,12 @@ """mark entity as being modified, fulltext reindex if needed""" raise NotImplementedError() - def delete_info(self, session, entity, uri, extid): - """delete system information on deletion of an entity by transfering - record from the entities table to the deleted_entities table + def delete_info_multi(self, session, entities, uri): + """delete system information on deletion of a list of entities with the + same etype and belinging to the same source """ raise NotImplementedError() - def delete_info_multi(self, session, entities, uri, extids): - """ame as delete_info but accepts a list of entities with - the same etype and belinging to the same source. - """ - for entity, extid in itertools.izip(entities, extids): - self.delete_info(session, entity, uri, extid) - def modified_entities(self, session, etypes, mtime): """return a 2-uple: * list of (etype, eid) of entities of the given types which have been diff -r cb0f4da64e86 -r 2983dd24494a server/sources/native.py --- a/server/sources/native.py Tue Jun 14 13:46:36 2011 +0200 +++ b/server/sources/native.py Tue Jun 14 15:37:09 2011 +0200 @@ -970,31 +970,13 @@ attrs = {'eid': entity.eid, 'mtime': datetime.now()} self.doexec(session, self.sqlgen.update('entities', attrs, ['eid']), attrs) - def delete_info(self, session, entity, uri, extid): - """delete system information on deletion of an entity: + def delete_info_multi(self, session, entities, uri): + """delete system information on deletion of a list of entities with the + same etype and belinging to the same source + * update the fti - * remove record from the entities table - * transfer it to the deleted_entities table if the entity's type is - multi-sources - """ - self.fti_unindex_entities(session, [entity]) - attrs = {'eid': entity.eid} - self.doexec(session, self.sqlgen.delete('entities', attrs), attrs) - if not entity.__regid__ in self.multisources_etypes: - return - if extid is not None: - assert isinstance(extid, str), type(extid) - extid = b64encode(extid) - attrs = {'type': entity.__regid__, 'eid': entity.eid, 'extid': extid, - 'source': uri, 'dtime': datetime.now()} - self.doexec(session, self.sqlgen.insert('deleted_entities', attrs), attrs) - - def delete_info_multi(self, session, entities, uri, extids): - """delete system information on deletion of an entity: - * update the fti - * remove record from the entities table - * transfer it to the deleted_entities table if the entity's type is - multi-sources + * remove record from the `entities` table + * transfer it to the `deleted_entities` """ self.fti_unindex_entities(session, entities) attrs = {'eid': '(%s)' % ','.join([str(_e.eid) for _e in entities])} @@ -1003,7 +985,8 @@ return attrs = {'type': entities[0].__regid__, 'source': uri, 'dtime': datetime.now()} - for entity, extid in itertools.izip(entities, extids): + for entity in entities: + extid = entity.cw_metainformation()['extid'] if extid is not None: assert isinstance(extid, str), type(extid) extid = b64encode(extid) diff -r cb0f4da64e86 -r 2983dd24494a server/ssplanner.py --- a/server/ssplanner.py Tue Jun 14 13:46:36 2011 +0200 +++ b/server/ssplanner.py Tue Jun 14 15:37:09 2011 +0200 @@ -28,7 +28,6 @@ from cubicweb.schema import VIRTUAL_RTYPES from cubicweb.rqlrewrite import add_types_restriction from cubicweb.server.session import security_enabled -from cubicweb.server.hook import CleanupDeletedEidsCacheOp from cubicweb.server.edition import EditedEntity READ_ONLY_RTYPES = set(('eid', 'has_text', 'is', 'is_instance_of', 'identity')) @@ -521,13 +520,7 @@ if results: todelete = frozenset(typed_eid(eid) for eid, in results) session = self.plan.session - # mark eids as being deleted in session info and setup cache update - # operation (register pending eids before actual deletion to avoid - # multiple call to glob_delete_entities) - op = CleanupDeletedEidsCacheOp.get_instance(session) - actual = todelete - op._container - op._container |= actual - session.repo.glob_delete_entities(session, actual) + session.repo.glob_delete_entities(session, todelete) return results class DeleteRelationsStep(Step):