[massive store] Ensure temporary metadata table get dropped
even if there is some integrity errors while inserting entity values or setting
back index/constraints, or any other errors. In case of exception, rollback to
ensure we're not committing undesired intermediate state.
--- a/cubicweb/dataimport/massive_store.py Wed Apr 19 10:09:25 2017 +0200
+++ b/cubicweb/dataimport/massive_store.py Wed Apr 19 15:09:31 2017 +0200
@@ -185,6 +185,19 @@
def finish(self):
"""Remove temporary tables and columns."""
+ try:
+ self._finish()
+ self._cnx.commit()
+ except Exception:
+ self._cnx.rollback()
+ raise
+ finally:
+ # delete the meta data table
+ self.sql('DROP TABLE IF EXISTS cwmassive_initialized')
+ self.commit()
+
+ def _finish(self):
+ """Remove temporary tables and columns."""
assert not self.slave_mode, 'finish method should only be called by the master store'
self.logger.info("Start cleaning")
# Get all the initialized etypes/rtypes
@@ -226,9 +239,6 @@
self._tmp_data_cleanup(tmp_tablename, rtype, uuid)
# restore all deleted indexes and constraints
self._dbh.restore_indexes_and_constraints()
- # delete the meta data table
- self.sql('DROP TABLE IF EXISTS cwmassive_initialized')
- self.commit()
def _insert_etype_metadata(self, etype, tmp_tablename):
"""Massive insertion of meta data for `etype`, with new entities in `tmp_tablename`.
--- a/cubicweb/dataimport/test/data-massimport/schema.py Wed Apr 19 10:09:25 2017 +0200
+++ b/cubicweb/dataimport/test/data-massimport/schema.py Wed Apr 19 15:09:31 2017 +0200
@@ -48,7 +48,7 @@
Entity type for timezone of geonames.
See timeZones.txt
"""
- code = String(maxsize=1024, indexed=True)
+ code = String(maxsize=1024, indexed=True, required=True)
gmt = Float()
dst = Float()
raw_offset = Float()
--- a/cubicweb/dataimport/test/test_massive_store.py Wed Apr 19 10:09:25 2017 +0200
+++ b/cubicweb/dataimport/test/test_massive_store.py Wed Apr 19 15:09:31 2017 +0200
@@ -16,6 +16,8 @@
# with this program. If not, see <http://www.gnu.org/licenses/>.
"""Massive store test case"""
+import psycopg2
+
from cubicweb.devtools import testlib, PostgresApptestConfiguration
from cubicweb.devtools import startpgcluster, stoppgcluster
from cubicweb.dataimport import ucsvreader, stores
@@ -113,7 +115,7 @@
def test_massimport_etype_metadata(self):
with self.admin_access.repo_cnx() as cnx:
store = MassiveObjectStore(cnx)
- timezone_eid = store.prepare_insert_entity('TimeZone')
+ timezone_eid = store.prepare_insert_entity('TimeZone', code=u'12')
store.prepare_insert_entity('Location', timezone=timezone_eid)
store.flush()
store.commit()
@@ -264,6 +266,16 @@
store.prepare_insert_entity('Location', name=u'toto')
store.finish()
+ def test_delete_metatable_on_integrity_error(self):
+ with self.admin_access.repo_cnx() as cnx:
+ store = MassiveObjectStore(cnx)
+ store.prepare_insert_entity('TimeZone')
+ store.flush()
+ store.commit()
+ with self.assertRaises(psycopg2.IntegrityError):
+ store.finish()
+ self.assertNotIn('cwmassive_initialized', set(self.get_db_descr(cnx)))
+
if __name__ == '__main__':
import unittest
--- a/cubicweb/dataimport/test/test_sqlgenstore.py Wed Apr 19 10:09:25 2017 +0200
+++ b/cubicweb/dataimport/test/test_sqlgenstore.py Wed Apr 19 15:09:31 2017 +0200
@@ -96,7 +96,7 @@
def test_sqlgenstore_etype_metadata(self):
with self.admin_access.repo_cnx() as cnx:
store = SQLGenObjectStore(cnx)
- timezone_eid = store.prepare_insert_entity('TimeZone')
+ timezone_eid = store.prepare_insert_entity('TimeZone', code=u'12')
store.prepare_insert_entity('Location', timezone=timezone_eid)
store.flush()
store.commit()