[dataimport] turn eids_seq_{start,range} into class attributes
authorJulien Cristau <julien.cristau@logilab.fr>
Mon, 09 Nov 2015 15:51:02 +0100
changeset 10872 ff4f94cfa2fb
parent 10871 1d4a94d04ec6
child 10873 0611466ce367
[dataimport] turn eids_seq_{start,range} into class attributes Only the master-mode store will actually use eids_seq_start to reset the eid sequence.
dataimport/massive_store.py
dataimport/test/test_massive_store.py
--- a/dataimport/massive_store.py	Mon Nov 09 15:29:07 2015 +0100
+++ b/dataimport/massive_store.py	Mon Nov 09 15:51:02 2015 +0100
@@ -81,11 +81,14 @@
        ...
        store.cleanup()
     """
+    # size of eid range reserved by the store for each batch
+    eids_seq_range = 10000
+    # initial eid (None means use the value in the db)
+    eids_seq_start = None
 
     def __init__(self, cnx, autoflush_metadata=True,
                  commit_at_flush=True,
                  iid_maxsize=1024, uri_param_name='rdf:about',
-                 eids_seq_range=10000, eids_seq_start=None,
                  on_commit_callback=None, on_rollback_callback=None,
                  slave_mode=False,
                  source=None):
@@ -96,11 +99,6 @@
                               Automatically flush the metadata after
                               each flush()
         - commit_at_flush: Boolean. Commit after each flush().
-        - eids_seq_range: Int. Range of the eids_seq_range to be fetched each time
-                               by the store (default is 10000).
-                               If None, the sequence eids is attached to each entity tables
-                               (backward compatibility with the 0.2.0).
-        - eids_seq_start: Int. Set the eids sequence value (if None, nothing is done).
         - iid_maxsize: Int. Max size of the iid, used to create the
                     iid_eid convertion table.
         - uri_param_name: String. If given, will use this parameter to get cw_uri
@@ -139,11 +137,9 @@
         # Initialized the meta tables of dataio for warm restart
         self._init_dataio_metatables()
         # Internal markers of initialization
-        self._eids_seq_range = eids_seq_range
-        self._eids_seq_start = eids_seq_start
-        if self._eids_seq_start is not None:
+        if self.eids_seq_start is not None and not self.slave_mode:
             self._cnx.system_sql(self._cnx.repo.system_source.dbhelper.sql_restart_numrange(
-                'entities_id_seq', initial_value=self._eids_seq_start + 1))
+                'entities_id_seq', initial_value=self.eids_seq_start + 1))
             cnx.commit()
         self.get_next_eid = lambda g=self._get_eid_gen(): next(g)
         # recreate then when self.finish() is called
@@ -377,7 +373,7 @@
         if etype not in self._initialized['entities']:
             # Only for non-initialized etype and not slave mode store
             if not self.slave_mode:
-                if self._eids_seq_range is None:
+                if self.eids_seq_range is None:
                     # Eids are directly set by the entities_id_seq.
                     # We attach this sequence to all the created etypes.
                     sql = ("ALTER TABLE cw_%s ALTER COLUMN cw_eid "
@@ -401,8 +397,8 @@
         a given number of eids from the 'entities_id_seq', and then
         storing them"""
         while True:
-            last_eid = self._cnx.repo.system_source.create_eid(self._cnx, self._eids_seq_range)
-            for eid in range(last_eid - self._eids_seq_range + 1, last_eid + 1):
+            last_eid = self._cnx.repo.system_source.create_eid(self._cnx, self.eids_seq_range)
+            for eid in range(last_eid - self.eids_seq_range + 1, last_eid + 1):
                 yield eid
 
     def _apply_size_constraints(self, etype, kwargs):
@@ -439,7 +435,7 @@
             else:
                 kwargs['cwuri'] = self._default_cwuri + str(self._count_cwuri)
                 self._count_cwuri += 1
-        if 'eid' not in kwargs and self._eids_seq_range is not None:
+        if 'eid' not in kwargs and self.eids_seq_range is not None:
             # If eid is not given and the eids sequence is set,
             # use the value from the sequence
             kwargs['eid'] = self.get_next_eid()
@@ -609,7 +605,7 @@
 
     def _cleanup_entities(self, etype):
         """ Cleanup etype table """
-        if self._eids_seq_range is None:
+        if self.eids_seq_range is None:
             # Remove DEFAULT eids sequence if added
             sql = 'ALTER TABLE cw_%s ALTER COLUMN cw_eid DROP DEFAULT;' % etype.lower()
             self.sql(sql)
--- a/dataimport/test/test_massive_store.py	Mon Nov 09 15:29:07 2015 +0100
+++ b/dataimport/test/test_massive_store.py	Mon Nov 09 15:51:02 2015 +0100
@@ -153,8 +153,12 @@
         self.assertIn('owned_by_relation_to_idx', indexes)
 
     def test_eids_seq_range(self):
+        class MyMassiveObjectStore(MassiveObjectStore):
+            eids_seq_range = 1000
+            eids_seq_start = 50000
+
         with self.admin_access.repo_cnx() as cnx:
-            store = MassiveObjectStore(cnx, eids_seq_range=1000, eids_seq_start=50000)
+            store = MyMassiveObjectStore(cnx)
             store.prepare_insert_entity('Location', name=u'toto')
             store.flush()
             cnx.commit()
@@ -163,15 +167,23 @@
             self.assertGreater(crs.fetchone()[0], 50000)
 
     def test_eid_entity(self):
+        class MyMassiveObjectStore(MassiveObjectStore):
+            eids_seq_range = 1000
+            eids_seq_start = 50000
+
         with self.admin_access.repo_cnx() as cnx:
-            store = MassiveObjectStore(cnx, eids_seq_range=1000, eids_seq_start=50000)
+            store = MyMassiveObjectStore(cnx)
             eid = store.prepare_insert_entity('Location', name=u'toto')
             store.flush()
             self.assertGreater(eid, 50000)
 
     def test_eid_entity_2(self):
+        class MyMassiveObjectStore(MassiveObjectStore):
+            eids_seq_range = 1000
+            eids_seq_start = 50000
+
         with self.admin_access.repo_cnx() as cnx:
-            store = MassiveObjectStore(cnx, eids_seq_range=1000, eids_seq_start=50000)
+            store = MyMassiveObjectStore(cnx)
             eid = store.prepare_insert_entity('Location', name=u'toto', eid=10000)
             store.flush()
         self.assertEqual(eid, 10000)