[dataimport] remove implicit commits from MassiveObjectStore
authorJulien Cristau <julien.cristau@logilab.fr>
Mon, 09 Nov 2015 16:43:50 +0100
changeset 10880 5fb592895e0f
parent 10879 3193d9ede8dd
child 10881 247ddae477ae
[dataimport] remove implicit commits from MassiveObjectStore Let the user decide when to commit the transaction.
dataimport/massive_store.py
dataimport/test/test_massive_store.py
--- a/dataimport/massive_store.py	Mon Nov 09 16:21:29 2015 +0100
+++ b/dataimport/massive_store.py	Mon Nov 09 16:43:50 2015 +0100
@@ -231,7 +231,6 @@
         self.sql('CREATE INDEX uri_eid_%(e)s_idx ON uri_eid_%(e)s' '(uri)' % {'e': etype.lower()})
         # Set the etype as converted
         self._uri_eid_inserted.add(etype)
-        self.commit()
 
     def convert_relations(self, etype_from, rtype, etype_to,
                           uri_label_from='cwuri', uri_label_to='cwuri'):
@@ -275,7 +274,6 @@
                             'ef': etype_from.lower() if etype_from else u''})
         except Exception as ex:
             self.logger.error("Can't insert relation %s: %s", rtype, ex)
-        self.commit()
 
     ### SQL UTILITIES #########################################################
 
@@ -461,7 +459,6 @@
                 self.sql('DROP TABLE %(r)s_relation_iid_tmp' % {'r': rtype})
             else:
                 self.logger.warning("inlined relation %s: no cleanup to be done for it" % rtype)
-        self.commit()
         # Get all the initialized etypes/rtypes
         if self._dbh.table_exists('cwmassive_initialized'):
             crs = self.sql('SELECT retype, type FROM cwmassive_initialized')
@@ -477,7 +474,6 @@
                          {'e': retype})
         # Create meta constraints (entities, is_instance_of, ...)
         self._create_metatables_constraints()
-        self.commit()
         # Delete the meta data table
         for table_name in ('cwmassive_initialized', 'cwmassive_constraints', 'cwmassive_metadata'):
             if self._dbh.table_exists(table_name):
@@ -576,8 +572,6 @@
                 self.insert_massive_meta_data(etype)
                 sql = 'INSERT INTO cwmassive_metadata VALUES (%(e)s)'
                 self.sql(sql, {'e': etype})
-        # Final commit
-        self.commit()
 
     def _cleanup_entities(self, etype):
         """ Cleanup etype table """
--- a/dataimport/test/test_massive_store.py	Mon Nov 09 16:21:29 2015 +0100
+++ b/dataimport/test/test_massive_store.py	Mon Nov 09 16:43:50 2015 +0100
@@ -210,6 +210,8 @@
             store = MassiveObjectStore(cnx)
             self.push_geonames_data(osp.join(HERE, 'data/geonames.csv'), store)
             store.flush()
+            store.commit()
+            store.finish()
         with self.admin_access.repo_cnx() as cnx:
             rset = cnx.execute('Any X WHERE X is Location')
             self.assertEqual(len(rset), 4000)