# HG changeset patch # User Julien Cristau # Date 1447081453 -3600 # Node ID fda5e42037a94a31fda09144d6bb4da2810b0969 # Parent a6a9a8fc94c308c65e8718cd8b0160a860788554 [dataimport] remove remaining references to dataio from MassiveObjectStore In particular, rename metadata tables to live in the ^cw.* namespace. diff -r a6a9a8fc94c3 -r fda5e42037a9 dataimport/massive_store.py --- a/dataimport/massive_store.py Mon Nov 09 15:55:45 2015 +0100 +++ b/dataimport/massive_store.py Mon Nov 09 16:04:13 2015 +0100 @@ -120,8 +120,8 @@ self._count_cwuri = 0 self.on_commit_callback = on_commit_callback self.on_rollback_callback = on_rollback_callback - # Initialized the meta tables of dataio for warm restart - self._init_dataio_metatables() + # Do our meta tables already exist? + self._init_massive_metatables() # Internal markers of initialization if self.eids_seq_start is not None and not self.slave_mode: self._cnx.system_sql(self._cnx.repo.system_source.dbhelper.sql_restart_numrange( @@ -175,13 +175,11 @@ # Add it to the initialized set self._initialized['init_uri_eid'].add(etype) - def _init_dataio_metatables(self): - """ Initialized the meta tables of dataio for warm restart - """ - # Check if dataio tables are not already created (i.e. a restart) - self._initialized_table_created = self._dbh.table_exists('dataio_initialized') - self._constraint_table_created = self._dbh.table_exists('dataio_constraints') - self._metadata_table_created = self._dbh.table_exists('dataio_metadata') + def _init_massive_metatables(self): + # Check if our tables are not already created (i.e. a restart) + self._initialized_table_created = self._dbh.table_exists('cwmassive_initialized') + self._constraint_table_created = self._dbh.table_exists('cwmassive_constraints') + self._metadata_table_created = self._dbh.table_exists('cwmassive_metadata') ### RELATE FUNCTION ####################################################### @@ -280,7 +278,7 @@ if not self._constraint_table_created: # Create a table to save the constraints # Allow reload even after crash - sql = "CREATE TABLE dataio_constraints (origtable text, query text, type varchar(256))" + sql = "CREATE TABLE cwmassive_constraints (origtable text, query text, type varchar(256))" self.sql(sql) self._constraint_table_created = True self._drop_table_constraints_indexes(tablename) @@ -289,26 +287,25 @@ """ Drop and store table constraints and indexes """ indexes, constraints = self._dbh.application_indexes_constraints(tablename) for name, query in constraints.items(): - sql = 'INSERT INTO dataio_constraints VALUES (%(e)s, %(c)s, %(t)s)' + sql = 'INSERT INTO cwmassive_constraints VALUES (%(e)s, %(c)s, %(t)s)' self.sql(sql, {'e': tablename, 'c': query, 't': 'constraint'}) sql = 'ALTER TABLE %s DROP CONSTRAINT %s CASCADE' % (tablename, name) self.sql(sql) for name, query in indexes.items(): - sql = 'INSERT INTO dataio_constraints VALUES (%(e)s, %(c)s, %(t)s)' + sql = 'INSERT INTO cwmassive_constraints VALUES (%(e)s, %(c)s, %(t)s)' self.sql(sql, {'e': tablename, 'c': query, 't': 'index'}) sql = 'DROP INDEX %s' % name self.sql(sql) def reapply_constraint_index(self, tablename): - if not self._dbh.table_exists('dataio_constraints'): - self.logger.info('The table dataio_constraints does not exist ' - '(keep_index option should be True)') + if not self._dbh.table_exists('cwmassive_constraints'): + self.logger.info('The table cwmassive_constraints does not exist') return - sql = 'SELECT query FROM dataio_constraints WHERE origtable = %(e)s' + sql = 'SELECT query FROM cwmassive_constraints WHERE origtable = %(e)s' crs = self.sql(sql, {'e': tablename}) for query, in crs.fetchall(): self.sql(query) - self.sql('DELETE FROM dataio_constraints WHERE origtable = %(e)s ' + self.sql('DELETE FROM cwmassive_constraints WHERE origtable = %(e)s ' 'AND query = %(q)s', {'e': tablename, 'q': query}) def _drop_metatables_constraints(self): @@ -337,16 +334,16 @@ self.drop_and_store_indexes_constraints(tablename) # Push the etype in the initialized table for easier restart self.init_create_initialized_table() - sql = 'INSERT INTO dataio_initialized VALUES (%(e)s, %(t)s)' + sql = 'INSERT INTO cwmassive_initialized VALUES (%(e)s, %(t)s)' self.sql(sql, {'e': rtype, 't': 'rtype'}) # Mark rtype as "initialized" for faster check self._initialized['rtypes'].add(rtype) def init_create_initialized_table(self): - """ Create the dataio initialized table + """ Create the cwmassive initialized table """ if not self._initialized_table_created: - sql = "CREATE TABLE dataio_initialized (retype text, type varchar(128))" + sql = "CREATE TABLE cwmassive_initialized (retype text, type varchar(128))" self.sql(sql) self._initialized_table_created = True @@ -367,7 +364,7 @@ self.drop_and_store_indexes_constraints(tablename) # Push the etype in the initialized table for easier restart self.init_create_initialized_table() - sql = 'INSERT INTO dataio_initialized VALUES (%(e)s, %(t)s)' + sql = 'INSERT INTO cwmassive_initialized VALUES (%(e)s, %(t)s)' self.sql(sql, {'e': etype, 't': 'etype'}) # Mark etype as "initialized" for faster check self._initialized['entities'].add(etype) @@ -460,8 +457,8 @@ self.logger.warning("inlined relation %s: no cleanup to be done for it" % rtype) self.commit() # Get all the initialized etypes/rtypes - if self._dbh.table_exists('dataio_initialized'): - crs = self.sql('SELECT retype, type FROM dataio_initialized') + if self._dbh.table_exists('cwmassive_initialized'): + crs = self.sql('SELECT retype, type FROM cwmassive_initialized') for retype, _type in crs.fetchall(): self.logger.info('Cleanup for %s' % retype) if _type == 'etype': @@ -470,13 +467,13 @@ elif _type == 'rtype': # Cleanup relations tables self._cleanup_relations(retype) - self.sql('DELETE FROM dataio_initialized WHERE retype = %(e)s', + self.sql('DELETE FROM cwmassive_initialized WHERE retype = %(e)s', {'e': retype}) # Create meta constraints (entities, is_instance_of, ...) self._create_metatables_constraints() self.commit() # Delete the meta data table - for table_name in ('dataio_initialized', 'dataio_constraints', 'dataio_metadata'): + for table_name in ('cwmassive_initialized', 'cwmassive_constraints', 'cwmassive_metadata'): if self._dbh.table_exists(table_name): self.sql('DROP TABLE %s' % table_name) self.commit() @@ -553,17 +550,17 @@ """ if self.slave_mode: raise RuntimeError('Flushing meta data is not allow in slave mode') - if not self._dbh.table_exists('dataio_initialized'): + if not self._dbh.table_exists('cwmassive_initialized'): self.logger.info('No information available for initialized etypes/rtypes') return if not self._metadata_table_created: # Keep the correctly flush meta data in database - sql = "CREATE TABLE dataio_metadata (etype text)" + sql = "CREATE TABLE cwmassive_metadata (etype text)" self.sql(sql) self._metadata_table_created = True - crs = self.sql('SELECT etype FROM dataio_metadata') + crs = self.sql('SELECT etype FROM cwmassive_metadata') already_flushed = set(e for e, in crs.fetchall()) - crs = self.sql('SELECT retype FROM dataio_initialized WHERE type = %(t)s', + crs = self.sql('SELECT retype FROM cwmassive_initialized WHERE type = %(t)s', {'t': 'etype'}) all_etypes = set(e for e, in crs.fetchall()) for etype in all_etypes: @@ -571,7 +568,7 @@ # Deals with meta data self.logger.info('Flushing meta data for %s' % etype) self.insert_massive_meta_data(etype) - sql = 'INSERT INTO dataio_metadata VALUES (%(e)s)' + sql = 'INSERT INTO cwmassive_metadata VALUES (%(e)s)' self.sql(sql, {'e': etype}) # Final commit self.commit()