144 self._initialized.add(etype) |
144 self._initialized.add(etype) |
145 self._dbh.drop_indexes('cw_%s' % etype.lower()) |
145 self._dbh.drop_indexes('cw_%s' % etype.lower()) |
146 self.sql('CREATE TABLE IF NOT EXISTS cwmassive_initialized' |
146 self.sql('CREATE TABLE IF NOT EXISTS cwmassive_initialized' |
147 '(retype text, type varchar(128))') |
147 '(retype text, type varchar(128))') |
148 self.sql("INSERT INTO cwmassive_initialized VALUES (%(e)s, 'etype')", {'e': etype}) |
148 self.sql("INSERT INTO cwmassive_initialized VALUES (%(e)s, 'etype')", {'e': etype}) |
149 self.sql('ALTER TABLE cw_%s ADD COLUMN extid VARCHAR(256)' % etype.lower()) |
|
150 attrs = self.metagen.base_etype_attrs(etype) |
149 attrs = self.metagen.base_etype_attrs(etype) |
151 data = copy(attrs) # base_etype_attrs is @cached, a copy is necessary |
150 data = copy(attrs) # base_etype_attrs is @cached, a copy is necessary |
152 data.update(kwargs) |
151 data.update(kwargs) |
153 if 'eid' not in data: |
152 if 'eid' not in data: |
154 # If eid is not given and the eids sequence is set, use the value from the sequence |
153 # If eid is not given and the eids sequence is set, use the value from the sequence |
156 data['eid'] = eid |
155 data['eid'] = eid |
157 # XXX default values could be set once for all in base entity |
156 # XXX default values could be set once for all in base entity |
158 default_values = self.default_values[etype] |
157 default_values = self.default_values[etype] |
159 missing_keys = set(default_values) - set(data) |
158 missing_keys = set(default_values) - set(data) |
160 data.update((key, default_values[key]) for key in missing_keys) |
159 data.update((key, default_values[key]) for key in missing_keys) |
161 extid = self.metagen.entity_extid(etype, data['eid'], data) |
|
162 if extid is not None: |
|
163 extid = b64encode(extid).decode('ascii') |
|
164 data['extid'] = extid |
|
165 self.metagen.init_entity_attrs(etype, data['eid'], data) |
160 self.metagen.init_entity_attrs(etype, data['eid'], data) |
166 self._data_entities[etype].append(data) |
161 self._data_entities[etype].append(data) |
167 return data['eid'] |
162 return data['eid'] |
168 |
163 |
169 def prepare_insert_relation(self, eid_from, rtype, eid_to, **kwargs): |
164 def prepare_insert_relation(self, eid_from, rtype, eid_to, **kwargs): |
201 # Get all the initialized etypes/rtypes |
196 # Get all the initialized etypes/rtypes |
202 if self._dbh.table_exists('cwmassive_initialized'): |
197 if self._dbh.table_exists('cwmassive_initialized'): |
203 cu = self.sql('SELECT retype, type FROM cwmassive_initialized') |
198 cu = self.sql('SELECT retype, type FROM cwmassive_initialized') |
204 for retype, _type in cu.fetchall(): |
199 for retype, _type in cu.fetchall(): |
205 self.logger.info('Cleanup for %s' % retype) |
200 self.logger.info('Cleanup for %s' % retype) |
206 if _type == 'etype': |
201 if _type == 'rtype': |
207 self.sql('ALTER TABLE cw_%s DROP COLUMN extid' % retype) |
|
208 elif _type == 'rtype': |
|
209 # Cleanup relations tables |
202 # Cleanup relations tables |
210 self._cleanup_relations(retype) |
203 self._cleanup_relations(retype) |
211 self.sql('DELETE FROM cwmassive_initialized WHERE retype = %(e)s', |
204 self.sql('DELETE FROM cwmassive_initialized WHERE retype = %(e)s', |
212 {'e': retype}) |
205 {'e': retype}) |
213 self._dbh.restore_indexes_and_constraints() |
206 self._dbh.restore_indexes_and_constraints() |
266 _data.append(_d) |
259 _data.append(_d) |
267 buf = pgstore._create_copyfrom_buffer(_data, columns) |
260 buf = pgstore._create_copyfrom_buffer(_data, columns) |
268 if not buf: |
261 if not buf: |
269 # The buffer is empty. This is probably due to error in _create_copyfrom_buffer |
262 # The buffer is empty. This is probably due to error in _create_copyfrom_buffer |
270 raise ValueError('Error in buffer creation for etype %s' % etype) |
263 raise ValueError('Error in buffer creation for etype %s' % etype) |
271 columns = ['cw_%s' % attr if attr != 'extid' else attr |
264 columns = ['cw_%s' % attr for attr in columns] |
272 for attr in columns] |
|
273 cursor = self._cnx.cnxset.cu |
265 cursor = self._cnx.cnxset.cu |
274 try: |
266 try: |
275 cursor.copy_from(buf, 'cw_%s' % etype.lower(), null='NULL', columns=columns) |
267 cursor.copy_from(buf, 'cw_%s' % etype.lower(), null='NULL', columns=columns) |
276 except Exception as exc: |
268 except Exception as exc: |
277 self.on_rollback(exc, etype, data) |
269 self.on_rollback(exc, etype, data) |
301 eschema = self.schema[etype] |
293 eschema = self.schema[etype] |
302 self._insert_meta_relation(etype, eschema.eid, 'is_relation') |
294 self._insert_meta_relation(etype, eschema.eid, 'is_relation') |
303 for parent_eschema in chain(eschema.ancestors(), [eschema]): |
295 for parent_eschema in chain(eschema.ancestors(), [eschema]): |
304 self._insert_meta_relation(etype, parent_eschema.eid, 'is_instance_of_relation') |
296 self._insert_meta_relation(etype, parent_eschema.eid, 'is_instance_of_relation') |
305 # finally insert records into the entities table |
297 # finally insert records into the entities table |
306 self.sql("INSERT INTO entities (eid, type, extid) " |
298 self.sql("INSERT INTO entities (eid, type) " |
307 "SELECT cw_eid, '%s', extid FROM cw_%s " |
299 "SELECT cw_eid, '%s' FROM cw_%s " |
308 "WHERE NOT EXISTS (SELECT 1 FROM entities WHERE eid=cw_eid)" |
300 "WHERE NOT EXISTS (SELECT 1 FROM entities WHERE eid=cw_eid)" |
309 % (etype, etype.lower())) |
301 % (etype, etype.lower())) |
310 |
302 |
311 def _insert_meta_relation(self, etype, eid_to, rtype): |
303 def _insert_meta_relation(self, etype, eid_to, rtype): |
312 self.sql("INSERT INTO %s (eid_from, eid_to) SELECT cw_eid, %s FROM cw_%s " |
304 self.sql("INSERT INTO %s (eid_from, eid_to) SELECT cw_eid, %s FROM cw_%s " |