351 self._cleanup_relations(retype) |
351 self._cleanup_relations(retype) |
352 self.sql('DELETE FROM cwmassive_initialized WHERE retype = %(e)s', |
352 self.sql('DELETE FROM cwmassive_initialized WHERE retype = %(e)s', |
353 {'e': retype}) |
353 {'e': retype}) |
354 self._dbh.restore_indexes_and_constraints() |
354 self._dbh.restore_indexes_and_constraints() |
355 # Delete the meta data table |
355 # Delete the meta data table |
356 for table_name in ('cwmassive_initialized', 'cwmassive_metadata'): |
356 self.sql('DROP TABLE IF EXISTS cwmassive_initialized') |
357 self.sql('DROP TABLE IF EXISTS %s' % table_name) |
|
358 self.commit() |
357 self.commit() |
359 |
358 |
360 # FLUSH ################################################################# |
359 # FLUSH ################################################################# |
361 |
360 |
362 def on_commit(self): |
361 def on_commit(self): |
419 cursor.copy_from(buf, 'cw_%s' % etype.lower(), null='NULL', columns=columns) |
418 cursor.copy_from(buf, 'cw_%s' % etype.lower(), null='NULL', columns=columns) |
420 except Exception as exc: |
419 except Exception as exc: |
421 self.on_rollback(exc, etype, data) |
420 self.on_rollback(exc, etype, data) |
422 # Clear data cache |
421 # Clear data cache |
423 self._data_entities[etype] = [] |
422 self._data_entities[etype] = [] |
424 if not self.slave_mode: |
423 if not self.slave_mode: |
425 self.flush_metadata() |
424 self._insert_etype_metadata(etype) |
426 |
|
427 def flush_metadata(self): |
|
428 """ Flush the meta data (entities table, is_instance table, ...) |
|
429 """ |
|
430 if self.slave_mode: |
|
431 raise RuntimeError('Flushing meta data is not allow in slave mode') |
|
432 if not self._dbh.table_exists('cwmassive_initialized'): |
|
433 self.logger.info('No information available for initialized etypes/rtypes') |
|
434 return |
|
435 # Keep the correctly flush meta data in database |
|
436 self.sql('CREATE TABLE IF NOT EXISTS cwmassive_metadata (etype text)') |
|
437 cu = self.sql('SELECT etype FROM cwmassive_metadata') |
|
438 already_flushed = set(e for e, in cu.fetchall()) |
|
439 cu = self.sql('SELECT retype FROM cwmassive_initialized WHERE type = %(t)s', |
|
440 {'t': 'etype'}) |
|
441 all_etypes = set(e for e, in cu.fetchall()) |
|
442 for etype in all_etypes: |
|
443 if etype not in already_flushed: |
|
444 # Deals with meta data |
|
445 self.logger.info('Flushing meta data for %s' % etype) |
|
446 self.insert_massive_metadata(etype) |
|
447 self.sql('INSERT INTO cwmassive_metadata VALUES (%(e)s)', {'e': etype}) |
|
448 |
425 |
449 def _cleanup_relations(self, rtype): |
426 def _cleanup_relations(self, rtype): |
450 """ Cleanup rtype table """ |
427 """ Cleanup rtype table """ |
451 # Push into relation table while removing duplicate |
428 # Push into relation table while removing duplicate |
452 self.sql('INSERT INTO %(r)s_relation (eid_from, eid_to) SELECT DISTINCT ' |
429 self.sql('INSERT INTO %(r)s_relation (eid_from, eid_to) SELECT DISTINCT ' |
454 'WHERE NOT EXISTS (SELECT 1 FROM %(r)s_relation AS TT WHERE ' |
431 'WHERE NOT EXISTS (SELECT 1 FROM %(r)s_relation AS TT WHERE ' |
455 'TT.eid_from=T.eid_from AND TT.eid_to=T.eid_to);' % {'r': rtype}) |
432 'TT.eid_from=T.eid_from AND TT.eid_to=T.eid_to);' % {'r': rtype}) |
456 # Drop temporary relation table |
433 # Drop temporary relation table |
457 self.sql('DROP TABLE %(r)s_relation_tmp' % {'r': rtype.lower()}) |
434 self.sql('DROP TABLE %(r)s_relation_tmp' % {'r': rtype.lower()}) |
458 |
435 |
459 def insert_massive_metadata(self, etype): |
436 def _insert_etype_metadata(self, etype): |
460 """ Massive insertion of meta data for a given etype, based on SQL statements. |
437 """Massive insertion of meta data for a given etype, based on SQL statements. |
461 """ |
438 """ |
462 # insert standard metadata relations |
439 # insert standard metadata relations |
463 for rtype, eid in self.metagen.base_etype_rels(etype).items(): |
440 for rtype, eid in self.metagen.base_etype_rels(etype).items(): |
464 self._insert_meta_relation(etype, eid, '%s_relation' % rtype) |
441 self._insert_meta_relation(etype, eid, '%s_relation' % rtype) |
465 # insert cw_source, is and is_instance_of relations (normally handled by the system source) |
442 # insert cw_source, is and is_instance_of relations (normally handled by the system source) |