[source/native] allow many eid creation per .create_eid call (closes #3526594)
[jcr: move migration code to bootstrapmigration_repository.py to make
sure we stop using the old sequence ASAP]
--- a/misc/migration/bootstrapmigration_repository.py Thu Feb 06 19:04:03 2014 +0100
+++ b/misc/migration/bootstrapmigration_repository.py Mon Mar 24 18:14:22 2014 +0100
@@ -35,6 +35,23 @@
ss.execschemarql(rql, rdef, ss.rdef2rql(rdef, CSTRMAP, groupmap=None))
commit(ask_confirm=False)
+def replace_eid_sequence_with_eid_numrange(session):
+ dbh = session.repo.system_source.dbhelper
+ cursor = session.cnxset.cu
+ try:
+ cursor.execute(dbh.sql_sequence_current_state('entities_id_seq'))
+ lasteid = cursor.fetchone()[0]
+ except: # programming error, already migrated
+ return
+
+ cursor.execute(dbh.sql_drop_sequence('entities_id_seq'))
+ cursor.execute(dbh.sql_create_numrange('entities_id_seq'))
+ cursor.execute(dbh.sql_restart_numrange('entities_id_seq', initial_value=lasteid))
+ session.commit()
+
+if applcubicwebversion < (3, 19, 0) and cubicwebversion >= (3, 19, 0):
+ replace_eid_sequence_with_eid_numrange(session)
+
if applcubicwebversion < (3, 17, 0) and cubicwebversion >= (3, 17, 0):
try:
add_cube('sioc', update_database=False)
--- a/server/sources/native.py Thu Feb 06 19:04:03 2014 +0100
+++ b/server/sources/native.py Mon Mar 24 18:14:22 2014 +0100
@@ -206,13 +206,14 @@
self.cnx.close()
self.cnx = None
- def create_eid(self, _session):
+ def create_eid(self, _session, count=1):
# lock needed to prevent 'Connection is busy with results for another
# command (0)' errors with SQLServer
+ assert count > 0
with self.lock:
- return self._create_eid() # pylint: disable=E1102
+ return self._create_eid(count)
- def _create_eid(self): # pylint: disable=E0202
+ def _create_eid(self, count):
# internal function doing the eid creation without locking.
# needed for the recursive handling of disconnections (otherwise we
# deadlock on self._eid_cnx_lock
@@ -222,20 +223,20 @@
cnx = self.cnx
try:
cursor = cnx.cursor()
- for sql in source.dbhelper.sqls_increment_sequence('entities_id_seq'):
+ for sql in source.dbhelper.sqls_increment_numrange('entities_id_seq', count):
cursor.execute(sql)
eid = cursor.fetchone()[0]
except (source.OperationalError, source.InterfaceError):
# FIXME: better detection of deconnection pb
source.warning("trying to reconnect create eid connection")
self.cnx = None
- return self._create_eid() # pylint: disable=E1102
+ return self._create_eid(count)
except source.DbapiError as exc:
# We get this one with pyodbc and SQL Server when connection was reset
if exc.args[0] == '08S01':
source.warning("trying to reconnect create eid connection")
self.cnx = None
- return self._create_eid() # pylint: disable=E1102
+ return self._create_eid(count)
else:
raise
except Exception: # WTF?
@@ -258,10 +259,11 @@
def close(self):
pass
- def create_eid(self, session):
+ def create_eid(self, session, count=1):
+ assert count > 0
source = self.source
with self.lock:
- for sql in source.dbhelper.sqls_increment_sequence('entities_id_seq'):
+ for sql in source.dbhelper.sqls_increment_numrange('entities_id_seq', count):
cursor = source.doexec(session, sql)
return cursor.fetchone()[0]
@@ -1421,7 +1423,7 @@
CREATE INDEX tx_relation_actions_eid_from_idx ON tx_relation_actions(eid_from);;
CREATE INDEX tx_relation_actions_eid_to_idx ON tx_relation_actions(eid_to);;
CREATE INDEX tx_relation_actions_tx_uuid_idx ON tx_relation_actions(tx_uuid);;
-""" % (helper.sql_create_sequence('entities_id_seq').replace(';', ';;'),
+""" % (helper.sql_create_numrange('entities_id_seq').replace(';', ';;'),
typemap['Datetime'],
typemap['Boolean'], typemap['Bytes'], typemap['Boolean'])
if helper.backend_name == 'sqlite':
@@ -1445,7 +1447,7 @@
DROP TABLE tx_entity_actions;
DROP TABLE tx_relation_actions;
DROP TABLE transactions;
-""" % helper.sql_drop_sequence('entities_id_seq')
+""" % helper.sql_drop_numrange('entities_id_seq')
def grant_schema(user, set_owner=True):
--- a/server/test/unittest_postgres.py Thu Feb 06 19:04:03 2014 +0100
+++ b/server/test/unittest_postgres.py Mon Mar 24 18:14:22 2014 +0100
@@ -1,4 +1,4 @@
-# copyright 2003-2012 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# copyright 2003-2014 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
@@ -17,6 +17,7 @@
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
from datetime import datetime
+from threading import Thread
from logilab.common.testlib import SkipTest
@@ -30,6 +31,26 @@
class PostgresFTITC(CubicWebTC):
configcls = PostgresApptestConfiguration
+ def test_eid_range(self):
+ # concurrent allocation of eid ranges
+ source = self.session.repo.sources_by_uri['system']
+ range1 = []
+ range2 = []
+ def allocate_eid_ranges(session, target):
+ for x in xrange(1, 10):
+ eid = source.create_eid(session, count=x)
+ target.extend(range(eid-x, eid))
+
+ t1 = Thread(target=lambda: allocate_eid_ranges(self.session, range1))
+ t2 = Thread(target=lambda: allocate_eid_ranges(self.session, range2))
+ t1.start()
+ t2.start()
+ t1.join()
+ t2.join()
+ self.assertEqual(range1, sorted(range1))
+ self.assertEqual(range2, sorted(range2))
+ self.assertEqual(set(), set(range1) & set(range2))
+
def test_occurence_count(self):
req = self.request()
c1 = req.create_entity('Card', title=u'c1',