# HG changeset patch # User Sylvain Thénault # Date 1269938356 -7200 # Node ID adc2122eed0322bb9dad1be214d471c6fd1ac778 # Parent bf5cbc351e997878782ccd188da27300ae588c4a [repo] more efficient eid cache operations handling based on set_operation; refactor * split CleanupEidTypeCache operation into one operation for deleted eids and another for added eids, and put them in cw.server.hook * for consistency, move fti unindexing handling from the repository to the system source * handling of deleted eids is done in plan execution (ssplanner), no needs to redo it in *delete_info -> no more need for _prepare_delete_info method, killed diff -r bf5cbc351e99 -r adc2122eed03 server/hook.py --- a/server/hook.py Tue Mar 30 10:28:41 2010 +0200 +++ b/server/hook.py Tue Mar 30 10:39:16 2010 +0200 @@ -548,3 +548,40 @@ execute = self.session.execute for rql in self.rqls: execute(*rql) + + +class CleanupNewEidsCacheOp(SingleLastOperation): + """on rollback of a insert query we have to remove from repository's + type/source cache eids of entities added in that transaction. + + NOTE: querier's rqlst/solutions cache may have been polluted too with + queries such as Any X WHERE X eid 32 if 32 has been rollbacked however + generated queries are unpredictable and analysing all the cache probably + too expensive. Notice that there is no pb when using args to specify eids + instead of giving them into the rql string. + """ + + def rollback_event(self): + """the observed connections pool has been rollbacked, + remove inserted eid from repository type/source cache + """ + try: + self.session.repo.clear_caches( + self.session.transaction_data['neweids']) + except KeyError: + pass + +class CleanupDeletedEidsCacheOp(SingleLastOperation): + """on commit of delete query, we have to remove from repository's + type/source cache eids of entities deleted in that transaction. + """ + + def commit_event(self): + """the observed connections pool has been rollbacked, + remove inserted eid from repository type/source cache + """ + try: + self.session.repo.clear_caches( + self.session.transaction_data['pendingeids']) + except KeyError: + pass diff -r bf5cbc351e99 -r adc2122eed03 server/repository.py --- a/server/repository.py Tue Mar 30 10:28:41 2010 +0200 +++ b/server/repository.py Tue Mar 30 10:39:16 2010 +0200 @@ -44,38 +44,6 @@ from cubicweb.server.session import Session, InternalSession, security_enabled -class CleanupEidTypeCacheOp(hook.SingleLastOperation): - """on rollback of a insert query or commit of delete query, we have to - clear repository's cache from no more valid entries - - NOTE: querier's rqlst/solutions cache may have been polluted too with - queries such as Any X WHERE X eid 32 if 32 has been rollbacked however - generated queries are unpredictable and analysing all the cache probably - too expensive. Notice that there is no pb when using args to specify eids - instead of giving them into the rql string. - """ - - def commit_event(self): - """the observed connections pool has been rollbacked, - remove inserted eid from repository type/source cache - """ - try: - self.session.repo.clear_caches( - self.session.transaction_data['pendingeids']) - except KeyError: - pass - - def rollback_event(self): - """the observed connections pool has been rollbacked, - remove inserted eid from repository type/source cache - """ - try: - self.session.repo.clear_caches( - self.session.transaction_data['neweids']) - except KeyError: - pass - - def del_existing_rel_if_needed(session, eidfrom, rtype, eidto): """delete existing relation when adding a new one if card is 1 or ? @@ -933,31 +901,20 @@ and index the entity with the full text index """ # begin by inserting eid/type/source/extid into the entities table - new = session.transaction_data.setdefault('neweids', set()) - new.add(entity.eid) + hook.set_operation(session, 'neweids', entity.eid, + hook.CleanupNewEidsCacheOp) self.system_source.add_info(session, entity, source, extid, complete) - CleanupEidTypeCacheOp(session) def delete_info(self, session, entity, sourceuri, extid): """called by external source when some entity known by the system source has been deleted in the external source """ - self._prepare_delete_info(session, entity, sourceuri) + # mark eid as being deleted in session info and setup cache update + # operation + hook.set_operation(session, 'pendingeids', entity.eid, + hook.CleanupDeletedEidsCacheOp) self._delete_info(session, entity, sourceuri, extid) - def _prepare_delete_info(self, session, entity, sourceuri): - """prepare the repository for deletion of an entity: - * update the fti - * mark eid as being deleted in session info - * setup cache update operation - * if undoable, get back all entity's attributes and relation - """ - eid = entity.eid - self.system_source.fti_unindex_entity(session, eid) - pending = session.transaction_data.setdefault('pendingeids', set()) - pending.add(eid) - CleanupEidTypeCacheOp(session) - def _delete_info(self, session, entity, sourceuri, extid): # attributes=None, relations=None): """delete system information on deletion of an entity: @@ -1150,7 +1107,6 @@ """delete an entity and all related entities from the repository""" entity = session.entity_from_eid(eid) etype, sourceuri, extid = self.type_and_source_from_eid(eid, session) - self._prepare_delete_info(session, entity, sourceuri) if server.DEBUG & server.DBG_REPO: print 'DELETE entity', etype, eid source = self.sources_by_uri[sourceuri] diff -r bf5cbc351e99 -r adc2122eed03 server/sources/native.py --- a/server/sources/native.py Tue Mar 30 10:28:41 2010 +0200 +++ b/server/sources/native.py Tue Mar 30 10:39:16 2010 +0200 @@ -670,10 +670,12 @@ def delete_info(self, session, entity, uri, extid): """delete system information on deletion of an entity: + * update the fti * remove record from the entities table * transfer it to the deleted_entities table if the entity's type is multi-sources """ + self.fti_unindex_entity(session, entity.eid) attrs = {'eid': entity.eid} self.doexec(session, self.sqlgen.delete('entities', attrs), attrs) if not entity.__regid__ in self.multisources_etypes: diff -r bf5cbc351e99 -r adc2122eed03 server/ssplanner.py --- a/server/ssplanner.py Tue Mar 30 10:28:41 2010 +0200 +++ b/server/ssplanner.py Tue Mar 30 10:39:16 2010 +0200 @@ -18,6 +18,7 @@ from cubicweb.schema import VIRTUAL_RTYPES from cubicweb.rqlrewrite import add_types_restriction from cubicweb.server.session import security_enabled +from cubicweb.server.hook import CleanupDeletedEidsCacheOp READ_ONLY_RTYPES = set(('eid', 'has_text', 'is', 'is_instance_of', 'identity')) @@ -510,8 +511,14 @@ todelete = frozenset(typed_eid(eid) for eid, in results) session = self.plan.session delete = session.repo.glob_delete_entity - # register pending eids first to avoid multiple deletion - pending = session.transaction_data.setdefault('pendingeids', set()) + # mark eids as being deleted in session info and setup cache update + # operation (register pending eids before actual deletion to avoid + # multiple call to glob_delete_entity) + try: + pending = session.transaction_data['pendingeids'] + except KeyError: + pending = session.transaction_data['pendingeids'] = set() + CleanupDeletedEidsCacheOp(session) actual = todelete - pending pending |= actual for eid in actual: