[hooks/operations] use set_operations for three ops (huge gains for massive imports) stable
authorAurelien Campeas <aurelien.campeas@logilab.fr>
Fri, 30 Apr 2010 16:39:50 +0200
branchstable
changeset 5448 9bf648d678cd
parent 5434 b56420abc00f
child 5449 a7e1b316af03
[hooks/operations] use set_operations for three ops (huge gains for massive imports)
hooks/integrity.py
hooks/metadata.py
hooks/security.py
--- a/hooks/integrity.py	Thu Apr 29 07:03:08 2010 +0200
+++ b/hooks/integrity.py	Fri Apr 30 16:39:50 2010 +0200
@@ -163,25 +163,28 @@
     """check a new relation satisfy its constraints
     """
     def precommit_event(self):
-        eidfrom, rtype, eidto = self.rdef
-        # first check related entities have not been deleted in the same
-        # transaction
-        if self.session.deleted_in_transaction(eidfrom):
-            return
-        if self.session.deleted_in_transaction(eidto):
-            return
-        for constraint in self.constraints:
-            # XXX
-            # * lock RQLConstraint as well?
-            # * use a constraint id to use per constraint lock and avoid
-            #   unnecessary commit serialization ?
-            if isinstance(constraint, RQLUniqueConstraint):
-                _acquire_unique_cstr_lock(self.session)
-            try:
-                constraint.repo_check(self.session, eidfrom, rtype, eidto)
-            except NotImplementedError:
-                self.critical('can\'t check constraint %s, not supported',
-                              constraint)
+        session = self.session
+        for values in session.transaction_data['check_constraints_op']:
+            eidfrom, rtype, eidto = values[:3]
+            # first check related entities have not been deleted in the same
+            # transaction
+            if session.deleted_in_transaction(eidfrom):
+                return
+            if session.deleted_in_transaction(eidto):
+                return
+            constraints = values[3:]
+            for constraint in constraints:
+                # XXX
+                # * lock RQLConstraint as well?
+                # * use a constraint id to use per constraint lock and avoid
+                #   unnecessary commit serialization ?
+                if isinstance(constraint, RQLUniqueConstraint):
+                    _acquire_unique_cstr_lock(session)
+                try:
+                    constraint.repo_check(session, eidfrom, rtype, eidto)
+                except NotImplementedError:
+                    self.critical('can\'t check constraint %s, not supported',
+                                  constraint)
 
     def commit_event(self):
         pass
@@ -201,8 +204,9 @@
         constraints = self._cw.schema_rproperty(self.rtype, self.eidfrom, self.eidto,
                                                 'constraints')
         if constraints:
-            _CheckConstraintsOp(self._cw, constraints=constraints,
-                               rdef=(self.eidfrom, self.rtype, self.eidto))
+            hook.set_operation(self._cw, 'check_constraints_op',
+                               (self.eidfrom, self.rtype, self.eidto) + tuple(constraints),
+                               _CheckConstraintsOp)
 
 
 class CheckAttributeConstraintHook(IntegrityHook):
@@ -221,8 +225,9 @@
                 constraints = [c for c in eschema.rdef(attr).constraints
                                if isinstance(c, (RQLUniqueConstraint, RQLConstraint))]
                 if constraints:
-                    _CheckConstraintsOp(self._cw, constraints=constraints,
-                                        rdef=(self.entity.eid, attr, None))
+                    hook.set_operation(self._cw, 'check_constraint_op',
+                                       (self.entity.eid, attr, None) + tuple(constraints),
+                                       _CheckConstraintsOp)
 
 
 class CheckUniqueHook(IntegrityHook):
--- a/hooks/metadata.py	Thu Apr 29 07:03:08 2010 +0200
+++ b/hooks/metadata.py	Fri Apr 30 16:39:50 2010 +0200
@@ -69,11 +69,13 @@
 
     def precommit_event(self):
         session = self.session
-        if session.deleted_in_transaction(self.entity.eid):
-            # entity have been created and deleted in the same transaction
-            return
-        if not self.entity.created_by:
-            session.add_relation(self.entity.eid, 'created_by', session.user.eid)
+        for eid in session.transaction_data['set_creator_op']:
+            if session.deleted_in_transaction(eid):
+                # entity have been created and deleted in the same transaction
+                continue
+            entity = session.entity_from_eid(eid)
+            if not entity.created_by:
+                session.add_relation(eid, 'created_by', session.user.eid)
 
 
 class SetIsHook(MetaDataHook):
@@ -108,14 +110,14 @@
     def __call__(self):
         if not self._cw.is_internal_session:
             self._cw.add_relation(self.entity.eid, 'owned_by', self._cw.user.eid)
-            _SetCreatorOp(self._cw, entity=self.entity)
-
+            hook.set_operation(self._cw, 'set_creator_op', self.entity.eid, _SetCreatorOp)
 
 class _SyncOwnersOp(hook.Operation):
     def precommit_event(self):
-        self.session.execute('SET X owned_by U WHERE C owned_by U, C eid %(c)s,'
-                             'NOT EXISTS(X owned_by U, X eid %(x)s)',
-                             {'c': self.compositeeid, 'x': self.composedeid})
+        for compositeeid, composedeid in self.session.transaction_data['sync_owners_op']:
+            self.session.execute('SET X owned_by U WHERE C owned_by U, C eid %(c)s,'
+                                 'NOT EXISTS(X owned_by U, X eid %(x)s)',
+                                 {'c': compositeeid, 'x': composedeid})
 
 
 class SyncCompositeOwner(MetaDataHook):
@@ -132,9 +134,9 @@
         eidfrom, eidto = self.eidfrom, self.eidto
         composite = self._cw.schema_rproperty(self.rtype, eidfrom, eidto, 'composite')
         if composite == 'subject':
-            _SyncOwnersOp(self._cw, compositeeid=eidfrom, composedeid=eidto)
+            hook.set_operation(self._cw, 'sync_owners_op', (eidfrom, eidto), _SyncOwnersOp)
         elif composite == 'object':
-            _SyncOwnersOp(self._cw, compositeeid=eidto, composedeid=eidfrom)
+            hook.set_operation(self._cw, 'sync_owners_op', (eidto, eidfrom), _SyncOwnersOp)
 
 
 class FixUserOwnershipHook(MetaDataHook):
--- a/hooks/security.py	Thu Apr 29 07:03:08 2010 +0200
+++ b/hooks/security.py	Fri Apr 30 16:39:50 2010 +0200
@@ -53,8 +53,12 @@
 class _CheckEntityPermissionOp(hook.LateOperation):
     def precommit_event(self):
         #print 'CheckEntityPermissionOp', self.session.user, self.entity, self.action
-        self.entity.check_perm(self.action)
-        check_entity_attributes(self.session, self.entity, self.editedattrs)
+        session = self.session
+        for values in session.transaction_data['check_entity_perm_op']:
+            entity = session.entity_from_eid(values[0])
+            action = values[1]
+        entity.check_perm(action)
+        check_entity_attributes(session, entity, values[2:])
 
     def commit_event(self):
         pass
@@ -89,9 +93,9 @@
     events = ('after_add_entity',)
 
     def __call__(self):
-        _CheckEntityPermissionOp(self._cw, entity=self.entity,
-                                 editedattrs=tuple(self.entity.edited_attributes),
-                                 action='add')
+        hook.set_operation(self._cw, 'check_entity_perm_op',
+                           (self.entity.eid, 'add') + tuple(self.entity.edited_attributes),
+                           _CheckEntityPermissionOp)
 
 
 class AfterUpdateEntitySecurityHook(SecurityHook):
@@ -108,9 +112,9 @@
             # save back editedattrs in case the entity is reedited later in the
             # same transaction, which will lead to edited_attributes being
             # overwritten
-            _CheckEntityPermissionOp(self._cw, entity=self.entity,
-                                     editedattrs=tuple(self.entity.edited_attributes),
-                                     action='update')
+            hook.set_operation(self._cw, 'check_entity_perm_op',
+                               (self.entity.eid, 'update') + tuple(self.entity.edited_attributes),
+                               _CheckEntityPermissionOp)
 
 
 class BeforeDelEntitySecurityHook(SecurityHook):