[source/native] allow many eid creation per .create_eid call (closes #3526594)
authorAurelien Campeas <aurelien.campeas@logilab.fr>
Mon, 24 Mar 2014 18:14:22 +0100
changeset 9585 3f5b59527d31
parent 9584 8209bede1a4b
child 9601 e5a80bd337e8
[source/native] allow many eid creation per .create_eid call (closes #3526594) [jcr: move migration code to bootstrapmigration_repository.py to make sure we stop using the old sequence ASAP]
misc/migration/bootstrapmigration_repository.py
server/sources/native.py
server/test/unittest_postgres.py
--- a/misc/migration/bootstrapmigration_repository.py	Thu Feb 06 19:04:03 2014 +0100
+++ b/misc/migration/bootstrapmigration_repository.py	Mon Mar 24 18:14:22 2014 +0100
@@ -35,6 +35,23 @@
     ss.execschemarql(rql, rdef, ss.rdef2rql(rdef, CSTRMAP, groupmap=None))
     commit(ask_confirm=False)
 
+def replace_eid_sequence_with_eid_numrange(session):
+    dbh = session.repo.system_source.dbhelper
+    cursor = session.cnxset.cu
+    try:
+        cursor.execute(dbh.sql_sequence_current_state('entities_id_seq'))
+        lasteid = cursor.fetchone()[0]
+    except: # programming error, already migrated
+        return
+
+    cursor.execute(dbh.sql_drop_sequence('entities_id_seq'))
+    cursor.execute(dbh.sql_create_numrange('entities_id_seq'))
+    cursor.execute(dbh.sql_restart_numrange('entities_id_seq', initial_value=lasteid))
+    session.commit()
+
+if applcubicwebversion < (3, 19, 0) and cubicwebversion >= (3, 19, 0):
+    replace_eid_sequence_with_eid_numrange(session)
+
 if applcubicwebversion < (3, 17, 0) and cubicwebversion >= (3, 17, 0):
     try:
         add_cube('sioc', update_database=False)
--- a/server/sources/native.py	Thu Feb 06 19:04:03 2014 +0100
+++ b/server/sources/native.py	Mon Mar 24 18:14:22 2014 +0100
@@ -206,13 +206,14 @@
             self.cnx.close()
         self.cnx = None
 
-    def create_eid(self, _session):
+    def create_eid(self, _session, count=1):
         # lock needed to prevent 'Connection is busy with results for another
         # command (0)' errors with SQLServer
+        assert count > 0
         with self.lock:
-            return self._create_eid() # pylint: disable=E1102
+            return self._create_eid(count)
 
-    def _create_eid(self): # pylint: disable=E0202
+    def _create_eid(self, count):
         # internal function doing the eid creation without locking.
         # needed for the recursive handling of disconnections (otherwise we
         # deadlock on self._eid_cnx_lock
@@ -222,20 +223,20 @@
         cnx = self.cnx
         try:
             cursor = cnx.cursor()
-            for sql in source.dbhelper.sqls_increment_sequence('entities_id_seq'):
+            for sql in source.dbhelper.sqls_increment_numrange('entities_id_seq', count):
                 cursor.execute(sql)
             eid = cursor.fetchone()[0]
         except (source.OperationalError, source.InterfaceError):
             # FIXME: better detection of deconnection pb
             source.warning("trying to reconnect create eid connection")
             self.cnx = None
-            return self._create_eid() # pylint: disable=E1102
+            return self._create_eid(count)
         except source.DbapiError as exc:
             # We get this one with pyodbc and SQL Server when connection was reset
             if exc.args[0] == '08S01':
                 source.warning("trying to reconnect create eid connection")
                 self.cnx = None
-                return self._create_eid() # pylint: disable=E1102
+                return self._create_eid(count)
             else:
                 raise
         except Exception: # WTF?
@@ -258,10 +259,11 @@
     def close(self):
         pass
 
-    def create_eid(self, session):
+    def create_eid(self, session, count=1):
+        assert count > 0
         source = self.source
         with self.lock:
-            for sql in source.dbhelper.sqls_increment_sequence('entities_id_seq'):
+            for sql in source.dbhelper.sqls_increment_numrange('entities_id_seq', count):
                 cursor = source.doexec(session, sql)
             return cursor.fetchone()[0]
 
@@ -1421,7 +1423,7 @@
 CREATE INDEX tx_relation_actions_eid_from_idx ON tx_relation_actions(eid_from);;
 CREATE INDEX tx_relation_actions_eid_to_idx ON tx_relation_actions(eid_to);;
 CREATE INDEX tx_relation_actions_tx_uuid_idx ON tx_relation_actions(tx_uuid);;
-""" % (helper.sql_create_sequence('entities_id_seq').replace(';', ';;'),
+""" % (helper.sql_create_numrange('entities_id_seq').replace(';', ';;'),
        typemap['Datetime'],
        typemap['Boolean'], typemap['Bytes'], typemap['Boolean'])
     if helper.backend_name == 'sqlite':
@@ -1445,7 +1447,7 @@
 DROP TABLE tx_entity_actions;
 DROP TABLE tx_relation_actions;
 DROP TABLE transactions;
-""" % helper.sql_drop_sequence('entities_id_seq')
+""" % helper.sql_drop_numrange('entities_id_seq')
 
 
 def grant_schema(user, set_owner=True):
--- a/server/test/unittest_postgres.py	Thu Feb 06 19:04:03 2014 +0100
+++ b/server/test/unittest_postgres.py	Mon Mar 24 18:14:22 2014 +0100
@@ -1,4 +1,4 @@
-# copyright 2003-2012 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# copyright 2003-2014 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
 # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
 #
 # This file is part of CubicWeb.
@@ -17,6 +17,7 @@
 # with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
 
 from datetime import datetime
+from threading import Thread
 
 from logilab.common.testlib import SkipTest
 
@@ -30,6 +31,26 @@
 class PostgresFTITC(CubicWebTC):
     configcls = PostgresApptestConfiguration
 
+    def test_eid_range(self):
+        # concurrent allocation of eid ranges
+        source = self.session.repo.sources_by_uri['system']
+        range1 = []
+        range2 = []
+        def allocate_eid_ranges(session, target):
+            for x in xrange(1, 10):
+                eid = source.create_eid(session, count=x)
+                target.extend(range(eid-x, eid))
+
+        t1 = Thread(target=lambda: allocate_eid_ranges(self.session, range1))
+        t2 = Thread(target=lambda: allocate_eid_ranges(self.session, range2))
+        t1.start()
+        t2.start()
+        t1.join()
+        t2.join()
+        self.assertEqual(range1, sorted(range1))
+        self.assertEqual(range2, sorted(range2))
+        self.assertEqual(set(), set(range1) & set(range2))
+
     def test_occurence_count(self):
         req = self.request()
         c1 = req.create_entity('Card', title=u'c1',