# HG changeset patch # User Aurelien Campeas # Date 1395681262 -3600 # Node ID 3f5b59527d3188f6e3430d6b42f6649c442cee82 # Parent 8209bede1a4b5477a18512fb0504867985732850 [source/native] allow many eid creation per .create_eid call (closes #3526594) [jcr: move migration code to bootstrapmigration_repository.py to make sure we stop using the old sequence ASAP] diff -r 8209bede1a4b -r 3f5b59527d31 misc/migration/bootstrapmigration_repository.py --- a/misc/migration/bootstrapmigration_repository.py Thu Feb 06 19:04:03 2014 +0100 +++ b/misc/migration/bootstrapmigration_repository.py Mon Mar 24 18:14:22 2014 +0100 @@ -35,6 +35,23 @@ ss.execschemarql(rql, rdef, ss.rdef2rql(rdef, CSTRMAP, groupmap=None)) commit(ask_confirm=False) +def replace_eid_sequence_with_eid_numrange(session): + dbh = session.repo.system_source.dbhelper + cursor = session.cnxset.cu + try: + cursor.execute(dbh.sql_sequence_current_state('entities_id_seq')) + lasteid = cursor.fetchone()[0] + except: # programming error, already migrated + return + + cursor.execute(dbh.sql_drop_sequence('entities_id_seq')) + cursor.execute(dbh.sql_create_numrange('entities_id_seq')) + cursor.execute(dbh.sql_restart_numrange('entities_id_seq', initial_value=lasteid)) + session.commit() + +if applcubicwebversion < (3, 19, 0) and cubicwebversion >= (3, 19, 0): + replace_eid_sequence_with_eid_numrange(session) + if applcubicwebversion < (3, 17, 0) and cubicwebversion >= (3, 17, 0): try: add_cube('sioc', update_database=False) diff -r 8209bede1a4b -r 3f5b59527d31 server/sources/native.py --- a/server/sources/native.py Thu Feb 06 19:04:03 2014 +0100 +++ b/server/sources/native.py Mon Mar 24 18:14:22 2014 +0100 @@ -206,13 +206,14 @@ self.cnx.close() self.cnx = None - def create_eid(self, _session): + def create_eid(self, _session, count=1): # lock needed to prevent 'Connection is busy with results for another # command (0)' errors with SQLServer + assert count > 0 with self.lock: - return self._create_eid() # pylint: disable=E1102 + return self._create_eid(count) - def _create_eid(self): # pylint: disable=E0202 + def _create_eid(self, count): # internal function doing the eid creation without locking. # needed for the recursive handling of disconnections (otherwise we # deadlock on self._eid_cnx_lock @@ -222,20 +223,20 @@ cnx = self.cnx try: cursor = cnx.cursor() - for sql in source.dbhelper.sqls_increment_sequence('entities_id_seq'): + for sql in source.dbhelper.sqls_increment_numrange('entities_id_seq', count): cursor.execute(sql) eid = cursor.fetchone()[0] except (source.OperationalError, source.InterfaceError): # FIXME: better detection of deconnection pb source.warning("trying to reconnect create eid connection") self.cnx = None - return self._create_eid() # pylint: disable=E1102 + return self._create_eid(count) except source.DbapiError as exc: # We get this one with pyodbc and SQL Server when connection was reset if exc.args[0] == '08S01': source.warning("trying to reconnect create eid connection") self.cnx = None - return self._create_eid() # pylint: disable=E1102 + return self._create_eid(count) else: raise except Exception: # WTF? @@ -258,10 +259,11 @@ def close(self): pass - def create_eid(self, session): + def create_eid(self, session, count=1): + assert count > 0 source = self.source with self.lock: - for sql in source.dbhelper.sqls_increment_sequence('entities_id_seq'): + for sql in source.dbhelper.sqls_increment_numrange('entities_id_seq', count): cursor = source.doexec(session, sql) return cursor.fetchone()[0] @@ -1421,7 +1423,7 @@ CREATE INDEX tx_relation_actions_eid_from_idx ON tx_relation_actions(eid_from);; CREATE INDEX tx_relation_actions_eid_to_idx ON tx_relation_actions(eid_to);; CREATE INDEX tx_relation_actions_tx_uuid_idx ON tx_relation_actions(tx_uuid);; -""" % (helper.sql_create_sequence('entities_id_seq').replace(';', ';;'), +""" % (helper.sql_create_numrange('entities_id_seq').replace(';', ';;'), typemap['Datetime'], typemap['Boolean'], typemap['Bytes'], typemap['Boolean']) if helper.backend_name == 'sqlite': @@ -1445,7 +1447,7 @@ DROP TABLE tx_entity_actions; DROP TABLE tx_relation_actions; DROP TABLE transactions; -""" % helper.sql_drop_sequence('entities_id_seq') +""" % helper.sql_drop_numrange('entities_id_seq') def grant_schema(user, set_owner=True): diff -r 8209bede1a4b -r 3f5b59527d31 server/test/unittest_postgres.py --- a/server/test/unittest_postgres.py Thu Feb 06 19:04:03 2014 +0100 +++ b/server/test/unittest_postgres.py Mon Mar 24 18:14:22 2014 +0100 @@ -1,4 +1,4 @@ -# copyright 2003-2012 LOGILAB S.A. (Paris, FRANCE), all rights reserved. +# copyright 2003-2014 LOGILAB S.A. (Paris, FRANCE), all rights reserved. # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr # # This file is part of CubicWeb. @@ -17,6 +17,7 @@ # with CubicWeb. If not, see . from datetime import datetime +from threading import Thread from logilab.common.testlib import SkipTest @@ -30,6 +31,26 @@ class PostgresFTITC(CubicWebTC): configcls = PostgresApptestConfiguration + def test_eid_range(self): + # concurrent allocation of eid ranges + source = self.session.repo.sources_by_uri['system'] + range1 = [] + range2 = [] + def allocate_eid_ranges(session, target): + for x in xrange(1, 10): + eid = source.create_eid(session, count=x) + target.extend(range(eid-x, eid)) + + t1 = Thread(target=lambda: allocate_eid_ranges(self.session, range1)) + t2 = Thread(target=lambda: allocate_eid_ranges(self.session, range2)) + t1.start() + t2.start() + t1.join() + t2.join() + self.assertEqual(range1, sorted(range1)) + self.assertEqual(range2, sorted(range2)) + self.assertEqual(set(), set(range1) & set(range2)) + def test_occurence_count(self): req = self.request() c1 = req.create_entity('Card', title=u'c1',