dataimport/massive_store.py
changeset 10880 5fb592895e0f
parent 10879 3193d9ede8dd
child 10881 247ddae477ae
equal deleted inserted replaced
10879:3193d9ede8dd 10880:5fb592895e0f
   229         self.sql(sql % {'l': uri_label, 'e': etype.lower()})
   229         self.sql(sql % {'l': uri_label, 'e': etype.lower()})
   230         # Add indexes
   230         # Add indexes
   231         self.sql('CREATE INDEX uri_eid_%(e)s_idx ON uri_eid_%(e)s' '(uri)' % {'e': etype.lower()})
   231         self.sql('CREATE INDEX uri_eid_%(e)s_idx ON uri_eid_%(e)s' '(uri)' % {'e': etype.lower()})
   232         # Set the etype as converted
   232         # Set the etype as converted
   233         self._uri_eid_inserted.add(etype)
   233         self._uri_eid_inserted.add(etype)
   234         self.commit()
       
   235 
   234 
   236     def convert_relations(self, etype_from, rtype, etype_to,
   235     def convert_relations(self, etype_from, rtype, etype_to,
   237                           uri_label_from='cwuri', uri_label_to='cwuri'):
   236                           uri_label_from='cwuri', uri_label_to='cwuri'):
   238         """ Flush the converted relations
   237         """ Flush the converted relations
   239         """
   238         """
   273             self.sql(sql % {'r': rtype.lower(),
   272             self.sql(sql % {'r': rtype.lower(),
   274                             'et': etype_to.lower() if etype_to else u'',
   273                             'et': etype_to.lower() if etype_to else u'',
   275                             'ef': etype_from.lower() if etype_from else u''})
   274                             'ef': etype_from.lower() if etype_from else u''})
   276         except Exception as ex:
   275         except Exception as ex:
   277             self.logger.error("Can't insert relation %s: %s", rtype, ex)
   276             self.logger.error("Can't insert relation %s: %s", rtype, ex)
   278         self.commit()
       
   279 
   277 
   280     ### SQL UTILITIES #########################################################
   278     ### SQL UTILITIES #########################################################
   281 
   279 
   282     def drop_and_store_indexes_constraints(self, tablename):
   280     def drop_and_store_indexes_constraints(self, tablename):
   283         # Drop indexes and constraints
   281         # Drop indexes and constraints
   459         for rtype in self._uri_rtypes:
   457         for rtype in self._uri_rtypes:
   460             if not self._cnx.repo.schema.rschema(rtype).inlined:
   458             if not self._cnx.repo.schema.rschema(rtype).inlined:
   461                 self.sql('DROP TABLE %(r)s_relation_iid_tmp' % {'r': rtype})
   459                 self.sql('DROP TABLE %(r)s_relation_iid_tmp' % {'r': rtype})
   462             else:
   460             else:
   463                 self.logger.warning("inlined relation %s: no cleanup to be done for it" % rtype)
   461                 self.logger.warning("inlined relation %s: no cleanup to be done for it" % rtype)
   464         self.commit()
       
   465         # Get all the initialized etypes/rtypes
   462         # Get all the initialized etypes/rtypes
   466         if self._dbh.table_exists('cwmassive_initialized'):
   463         if self._dbh.table_exists('cwmassive_initialized'):
   467             crs = self.sql('SELECT retype, type FROM cwmassive_initialized')
   464             crs = self.sql('SELECT retype, type FROM cwmassive_initialized')
   468             for retype, _type in crs.fetchall():
   465             for retype, _type in crs.fetchall():
   469                 self.logger.info('Cleanup for %s' % retype)
   466                 self.logger.info('Cleanup for %s' % retype)
   475                     self._cleanup_relations(retype)
   472                     self._cleanup_relations(retype)
   476                 self.sql('DELETE FROM cwmassive_initialized WHERE retype = %(e)s',
   473                 self.sql('DELETE FROM cwmassive_initialized WHERE retype = %(e)s',
   477                          {'e': retype})
   474                          {'e': retype})
   478         # Create meta constraints (entities, is_instance_of, ...)
   475         # Create meta constraints (entities, is_instance_of, ...)
   479         self._create_metatables_constraints()
   476         self._create_metatables_constraints()
   480         self.commit()
       
   481         # Delete the meta data table
   477         # Delete the meta data table
   482         for table_name in ('cwmassive_initialized', 'cwmassive_constraints', 'cwmassive_metadata'):
   478         for table_name in ('cwmassive_initialized', 'cwmassive_constraints', 'cwmassive_metadata'):
   483             if self._dbh.table_exists(table_name):
   479             if self._dbh.table_exists(table_name):
   484                 self.sql('DROP TABLE %s' % table_name)
   480                 self.sql('DROP TABLE %s' % table_name)
   485         self.commit()
   481         self.commit()
   574                 # Deals with meta data
   570                 # Deals with meta data
   575                 self.logger.info('Flushing meta data for %s' % etype)
   571                 self.logger.info('Flushing meta data for %s' % etype)
   576                 self.insert_massive_meta_data(etype)
   572                 self.insert_massive_meta_data(etype)
   577                 sql = 'INSERT INTO cwmassive_metadata VALUES (%(e)s)'
   573                 sql = 'INSERT INTO cwmassive_metadata VALUES (%(e)s)'
   578                 self.sql(sql, {'e': etype})
   574                 self.sql(sql, {'e': etype})
   579         # Final commit
       
   580         self.commit()
       
   581 
   575 
   582     def _cleanup_entities(self, etype):
   576     def _cleanup_entities(self, etype):
   583         """ Cleanup etype table """
   577         """ Cleanup etype table """
   584         if self.eids_seq_range is None:
   578         if self.eids_seq_range is None:
   585             # Remove DEFAULT eids sequence if added
   579             # Remove DEFAULT eids sequence if added