133 self.action_methods = { |
133 self.action_methods = { |
134 'copy': self.related_copy, |
134 'copy': self.related_copy, |
135 'link-or-create': self.related_link_or_create, |
135 'link-or-create': self.related_link_or_create, |
136 'link': self.related_link, |
136 'link': self.related_link, |
137 } |
137 } |
|
138 self._parsed_urls = {} |
|
139 self._processed_entities = set() |
138 |
140 |
139 # mapping handling ######################################################### |
141 # mapping handling ######################################################### |
140 |
142 |
141 def add_schema_config(self, schemacfg, checkonly=False): |
143 def add_schema_config(self, schemacfg, checkonly=False): |
142 """added CWSourceSchemaConfig, modify mapping accordingly""" |
144 """added CWSourceSchemaConfig, modify mapping accordingly""" |
194 def process_item(self, item, rels): |
196 def process_item(self, item, rels): |
195 entity = self.extid2entity(str(item.pop('cwuri')), item.pop('cwtype'), |
197 entity = self.extid2entity(str(item.pop('cwuri')), item.pop('cwtype'), |
196 item=item) |
198 item=item) |
197 if entity is None: |
199 if entity is None: |
198 return None |
200 return None |
|
201 if entity.eid in self._processed_entities: |
|
202 return entity |
|
203 self._processed_entities.add(entity.eid) |
199 if not (self.created_during_pull(entity) or self.updated_during_pull(entity)): |
204 if not (self.created_during_pull(entity) or self.updated_during_pull(entity)): |
200 self.notify_updated(entity) |
205 self.notify_updated(entity) |
201 item.pop('eid') |
206 item.pop('eid') |
202 # XXX check modification date |
207 # XXX check modification date |
203 attrs = extract_typed_attrs(entity.e_schema, item) |
208 attrs = extract_typed_attrs(entity.e_schema, item) |
294 self._set_relation(entity, rtype, role, eids) |
299 self._set_relation(entity, rtype, role, eids) |
295 else: |
300 else: |
296 self._clear_relation(entity, rtype, role, (ttype,)) |
301 self._clear_relation(entity, rtype, role, (ttype,)) |
297 |
302 |
298 def _complete_item(self, item, add_relations=True): |
303 def _complete_item(self, item, add_relations=True): |
299 itemurl = item['cwuri'] + '?vid=xml' |
304 try: |
300 if add_relations: |
305 return self._parsed_urls[(item['cwuri'], add_relations)] |
301 for rtype, role, _ in self.source.mapping.get(item['cwtype'], ()): |
306 except KeyError: |
302 itemurl += '&relation=%s_%s' % (rtype, role) |
307 itemurl = item['cwuri'] + '?vid=xml' |
303 item_rels = list(self.parse(itemurl)) |
308 if add_relations: |
304 assert len(item_rels) == 1 |
309 for rtype, role, _ in self.source.mapping.get(item['cwtype'], ()): |
305 return item_rels[0] |
310 itemurl += '&relation=%s_%s' % (rtype, role) |
|
311 item_rels = list(self.parse(itemurl)) |
|
312 assert len(item_rels) == 1, 'url %s expected to bring back one '\ |
|
313 'and only one entity, got %s' % (itemurl, len(item_rels)) |
|
314 self._parsed_urls[(item['cwuri'], add_relations)] = item_rels[0] |
|
315 return item_rels[0] |
306 |
316 |
307 def _clear_relation(self, entity, rtype, role, ttypes): |
317 def _clear_relation(self, entity, rtype, role, ttypes): |
308 if entity.eid not in self.stats['created']: |
318 if entity.eid not in self.stats['created']: |
309 if len(ttypes) > 1: |
319 if len(ttypes) > 1: |
310 typerestr = ', Y is IN(%s)' % ','.join(ttypes) |
320 typerestr = ', Y is IN(%s)' % ','.join(ttypes) |