[dataimport] remove implicit commits from MassiveObjectStore
Let the user decide when to commit the transaction.
--- a/dataimport/massive_store.py Mon Nov 09 16:21:29 2015 +0100
+++ b/dataimport/massive_store.py Mon Nov 09 16:43:50 2015 +0100
@@ -231,7 +231,6 @@
self.sql('CREATE INDEX uri_eid_%(e)s_idx ON uri_eid_%(e)s' '(uri)' % {'e': etype.lower()})
# Set the etype as converted
self._uri_eid_inserted.add(etype)
- self.commit()
def convert_relations(self, etype_from, rtype, etype_to,
uri_label_from='cwuri', uri_label_to='cwuri'):
@@ -275,7 +274,6 @@
'ef': etype_from.lower() if etype_from else u''})
except Exception as ex:
self.logger.error("Can't insert relation %s: %s", rtype, ex)
- self.commit()
### SQL UTILITIES #########################################################
@@ -461,7 +459,6 @@
self.sql('DROP TABLE %(r)s_relation_iid_tmp' % {'r': rtype})
else:
self.logger.warning("inlined relation %s: no cleanup to be done for it" % rtype)
- self.commit()
# Get all the initialized etypes/rtypes
if self._dbh.table_exists('cwmassive_initialized'):
crs = self.sql('SELECT retype, type FROM cwmassive_initialized')
@@ -477,7 +474,6 @@
{'e': retype})
# Create meta constraints (entities, is_instance_of, ...)
self._create_metatables_constraints()
- self.commit()
# Delete the meta data table
for table_name in ('cwmassive_initialized', 'cwmassive_constraints', 'cwmassive_metadata'):
if self._dbh.table_exists(table_name):
@@ -576,8 +572,6 @@
self.insert_massive_meta_data(etype)
sql = 'INSERT INTO cwmassive_metadata VALUES (%(e)s)'
self.sql(sql, {'e': etype})
- # Final commit
- self.commit()
def _cleanup_entities(self, etype):
""" Cleanup etype table """
--- a/dataimport/test/test_massive_store.py Mon Nov 09 16:21:29 2015 +0100
+++ b/dataimport/test/test_massive_store.py Mon Nov 09 16:43:50 2015 +0100
@@ -210,6 +210,8 @@
store = MassiveObjectStore(cnx)
self.push_geonames_data(osp.join(HERE, 'data/geonames.csv'), store)
store.flush()
+ store.commit()
+ store.finish()
with self.admin_access.repo_cnx() as cnx:
rset = cnx.execute('Any X WHERE X is Location')
self.assertEqual(len(rset), 4000)