cubicweb/dataimport/massive_store.py
changeset 11329 a8cab8fb54ba
parent 11328 9f2d7da47526
child 11330 6adfa1e75179
equal deleted inserted replaced
11328:9f2d7da47526 11329:a8cab8fb54ba
   351                     self._cleanup_relations(retype)
   351                     self._cleanup_relations(retype)
   352                 self.sql('DELETE FROM cwmassive_initialized WHERE retype = %(e)s',
   352                 self.sql('DELETE FROM cwmassive_initialized WHERE retype = %(e)s',
   353                          {'e': retype})
   353                          {'e': retype})
   354         self._dbh.restore_indexes_and_constraints()
   354         self._dbh.restore_indexes_and_constraints()
   355         # Delete the meta data table
   355         # Delete the meta data table
   356         for table_name in ('cwmassive_initialized', 'cwmassive_metadata'):
   356         self.sql('DROP TABLE IF EXISTS cwmassive_initialized')
   357             self.sql('DROP TABLE IF EXISTS %s' % table_name)
       
   358         self.commit()
   357         self.commit()
   359 
   358 
   360     # FLUSH #################################################################
   359     # FLUSH #################################################################
   361 
   360 
   362     def on_commit(self):
   361     def on_commit(self):
   419                 cursor.copy_from(buf, 'cw_%s' % etype.lower(), null='NULL', columns=columns)
   418                 cursor.copy_from(buf, 'cw_%s' % etype.lower(), null='NULL', columns=columns)
   420             except Exception as exc:
   419             except Exception as exc:
   421                 self.on_rollback(exc, etype, data)
   420                 self.on_rollback(exc, etype, data)
   422             # Clear data cache
   421             # Clear data cache
   423             self._data_entities[etype] = []
   422             self._data_entities[etype] = []
   424         if not self.slave_mode:
   423             if not self.slave_mode:
   425             self.flush_metadata()
   424                 self._insert_etype_metadata(etype)
   426 
       
   427     def flush_metadata(self):
       
   428         """ Flush the meta data (entities table, is_instance table, ...)
       
   429         """
       
   430         if self.slave_mode:
       
   431             raise RuntimeError('Flushing meta data is not allow in slave mode')
       
   432         if not self._dbh.table_exists('cwmassive_initialized'):
       
   433             self.logger.info('No information available for initialized etypes/rtypes')
       
   434             return
       
   435         # Keep the correctly flush meta data in database
       
   436         self.sql('CREATE TABLE IF NOT EXISTS cwmassive_metadata (etype text)')
       
   437         cu = self.sql('SELECT etype FROM cwmassive_metadata')
       
   438         already_flushed = set(e for e, in cu.fetchall())
       
   439         cu = self.sql('SELECT retype FROM cwmassive_initialized WHERE type = %(t)s',
       
   440                       {'t': 'etype'})
       
   441         all_etypes = set(e for e, in cu.fetchall())
       
   442         for etype in all_etypes:
       
   443             if etype not in already_flushed:
       
   444                 # Deals with meta data
       
   445                 self.logger.info('Flushing meta data for %s' % etype)
       
   446                 self.insert_massive_metadata(etype)
       
   447                 self.sql('INSERT INTO cwmassive_metadata VALUES (%(e)s)', {'e': etype})
       
   448 
   425 
   449     def _cleanup_relations(self, rtype):
   426     def _cleanup_relations(self, rtype):
   450         """ Cleanup rtype table """
   427         """ Cleanup rtype table """
   451         # Push into relation table while removing duplicate
   428         # Push into relation table while removing duplicate
   452         self.sql('INSERT INTO %(r)s_relation (eid_from, eid_to) SELECT DISTINCT '
   429         self.sql('INSERT INTO %(r)s_relation (eid_from, eid_to) SELECT DISTINCT '
   454                  'WHERE NOT EXISTS (SELECT 1 FROM %(r)s_relation AS TT WHERE '
   431                  'WHERE NOT EXISTS (SELECT 1 FROM %(r)s_relation AS TT WHERE '
   455                  'TT.eid_from=T.eid_from AND TT.eid_to=T.eid_to);' % {'r': rtype})
   432                  'TT.eid_from=T.eid_from AND TT.eid_to=T.eid_to);' % {'r': rtype})
   456         # Drop temporary relation table
   433         # Drop temporary relation table
   457         self.sql('DROP TABLE %(r)s_relation_tmp' % {'r': rtype.lower()})
   434         self.sql('DROP TABLE %(r)s_relation_tmp' % {'r': rtype.lower()})
   458 
   435 
   459     def insert_massive_metadata(self, etype):
   436     def _insert_etype_metadata(self, etype):
   460         """ Massive insertion of meta data for a given etype, based on SQL statements.
   437         """Massive insertion of meta data for a given etype, based on SQL statements.
   461         """
   438         """
   462         # insert standard metadata relations
   439         # insert standard metadata relations
   463         for rtype, eid in self.metagen.base_etype_rels(etype).items():
   440         for rtype, eid in self.metagen.base_etype_rels(etype).items():
   464             self._insert_meta_relation(etype, eid, '%s_relation' % rtype)
   441             self._insert_meta_relation(etype, eid, '%s_relation' % rtype)
   465         # insert cw_source, is and is_instance_of relations (normally handled by the system source)
   442         # insert cw_source, is and is_instance_of relations (normally handled by the system source)