[repo] more efficient eid cache operations handling based on set_operation; refactor
* split CleanupEidTypeCache operation into one operation for deleted eids
and another for added eids, and put them in cw.server.hook
* for consistency, move fti unindexing handling from the repository
to the system source
* handling of deleted eids is done in plan execution (ssplanner), no
needs to redo it in *delete_info
-> no more need for _prepare_delete_info method, killed
--- a/server/hook.py Tue Mar 30 10:28:41 2010 +0200
+++ b/server/hook.py Tue Mar 30 10:39:16 2010 +0200
@@ -548,3 +548,40 @@
execute = self.session.execute
for rql in self.rqls:
execute(*rql)
+
+
+class CleanupNewEidsCacheOp(SingleLastOperation):
+ """on rollback of a insert query we have to remove from repository's
+ type/source cache eids of entities added in that transaction.
+
+ NOTE: querier's rqlst/solutions cache may have been polluted too with
+ queries such as Any X WHERE X eid 32 if 32 has been rollbacked however
+ generated queries are unpredictable and analysing all the cache probably
+ too expensive. Notice that there is no pb when using args to specify eids
+ instead of giving them into the rql string.
+ """
+
+ def rollback_event(self):
+ """the observed connections pool has been rollbacked,
+ remove inserted eid from repository type/source cache
+ """
+ try:
+ self.session.repo.clear_caches(
+ self.session.transaction_data['neweids'])
+ except KeyError:
+ pass
+
+class CleanupDeletedEidsCacheOp(SingleLastOperation):
+ """on commit of delete query, we have to remove from repository's
+ type/source cache eids of entities deleted in that transaction.
+ """
+
+ def commit_event(self):
+ """the observed connections pool has been rollbacked,
+ remove inserted eid from repository type/source cache
+ """
+ try:
+ self.session.repo.clear_caches(
+ self.session.transaction_data['pendingeids'])
+ except KeyError:
+ pass
--- a/server/repository.py Tue Mar 30 10:28:41 2010 +0200
+++ b/server/repository.py Tue Mar 30 10:39:16 2010 +0200
@@ -44,38 +44,6 @@
from cubicweb.server.session import Session, InternalSession, security_enabled
-class CleanupEidTypeCacheOp(hook.SingleLastOperation):
- """on rollback of a insert query or commit of delete query, we have to
- clear repository's cache from no more valid entries
-
- NOTE: querier's rqlst/solutions cache may have been polluted too with
- queries such as Any X WHERE X eid 32 if 32 has been rollbacked however
- generated queries are unpredictable and analysing all the cache probably
- too expensive. Notice that there is no pb when using args to specify eids
- instead of giving them into the rql string.
- """
-
- def commit_event(self):
- """the observed connections pool has been rollbacked,
- remove inserted eid from repository type/source cache
- """
- try:
- self.session.repo.clear_caches(
- self.session.transaction_data['pendingeids'])
- except KeyError:
- pass
-
- def rollback_event(self):
- """the observed connections pool has been rollbacked,
- remove inserted eid from repository type/source cache
- """
- try:
- self.session.repo.clear_caches(
- self.session.transaction_data['neweids'])
- except KeyError:
- pass
-
-
def del_existing_rel_if_needed(session, eidfrom, rtype, eidto):
"""delete existing relation when adding a new one if card is 1 or ?
@@ -933,31 +901,20 @@
and index the entity with the full text index
"""
# begin by inserting eid/type/source/extid into the entities table
- new = session.transaction_data.setdefault('neweids', set())
- new.add(entity.eid)
+ hook.set_operation(session, 'neweids', entity.eid,
+ hook.CleanupNewEidsCacheOp)
self.system_source.add_info(session, entity, source, extid, complete)
- CleanupEidTypeCacheOp(session)
def delete_info(self, session, entity, sourceuri, extid):
"""called by external source when some entity known by the system source
has been deleted in the external source
"""
- self._prepare_delete_info(session, entity, sourceuri)
+ # mark eid as being deleted in session info and setup cache update
+ # operation
+ hook.set_operation(session, 'pendingeids', entity.eid,
+ hook.CleanupDeletedEidsCacheOp)
self._delete_info(session, entity, sourceuri, extid)
- def _prepare_delete_info(self, session, entity, sourceuri):
- """prepare the repository for deletion of an entity:
- * update the fti
- * mark eid as being deleted in session info
- * setup cache update operation
- * if undoable, get back all entity's attributes and relation
- """
- eid = entity.eid
- self.system_source.fti_unindex_entity(session, eid)
- pending = session.transaction_data.setdefault('pendingeids', set())
- pending.add(eid)
- CleanupEidTypeCacheOp(session)
-
def _delete_info(self, session, entity, sourceuri, extid):
# attributes=None, relations=None):
"""delete system information on deletion of an entity:
@@ -1150,7 +1107,6 @@
"""delete an entity and all related entities from the repository"""
entity = session.entity_from_eid(eid)
etype, sourceuri, extid = self.type_and_source_from_eid(eid, session)
- self._prepare_delete_info(session, entity, sourceuri)
if server.DEBUG & server.DBG_REPO:
print 'DELETE entity', etype, eid
source = self.sources_by_uri[sourceuri]
--- a/server/sources/native.py Tue Mar 30 10:28:41 2010 +0200
+++ b/server/sources/native.py Tue Mar 30 10:39:16 2010 +0200
@@ -670,10 +670,12 @@
def delete_info(self, session, entity, uri, extid):
"""delete system information on deletion of an entity:
+ * update the fti
* remove record from the entities table
* transfer it to the deleted_entities table if the entity's type is
multi-sources
"""
+ self.fti_unindex_entity(session, entity.eid)
attrs = {'eid': entity.eid}
self.doexec(session, self.sqlgen.delete('entities', attrs), attrs)
if not entity.__regid__ in self.multisources_etypes:
--- a/server/ssplanner.py Tue Mar 30 10:28:41 2010 +0200
+++ b/server/ssplanner.py Tue Mar 30 10:39:16 2010 +0200
@@ -18,6 +18,7 @@
from cubicweb.schema import VIRTUAL_RTYPES
from cubicweb.rqlrewrite import add_types_restriction
from cubicweb.server.session import security_enabled
+from cubicweb.server.hook import CleanupDeletedEidsCacheOp
READ_ONLY_RTYPES = set(('eid', 'has_text', 'is', 'is_instance_of', 'identity'))
@@ -510,8 +511,14 @@
todelete = frozenset(typed_eid(eid) for eid, in results)
session = self.plan.session
delete = session.repo.glob_delete_entity
- # register pending eids first to avoid multiple deletion
- pending = session.transaction_data.setdefault('pendingeids', set())
+ # mark eids as being deleted in session info and setup cache update
+ # operation (register pending eids before actual deletion to avoid
+ # multiple call to glob_delete_entity)
+ try:
+ pending = session.transaction_data['pendingeids']
+ except KeyError:
+ pending = session.transaction_data['pendingeids'] = set()
+ CleanupDeletedEidsCacheOp(session)
actual = todelete - pending
pending |= actual
for eid in actual: