server/sources/datafeed.py
changeset 8430 5bee87a14bb1
parent 8429 cad2d8e03b33
child 8434 39c5bb4dcc59
--- a/server/sources/datafeed.py	Fri May 25 17:18:00 2012 +0200
+++ b/server/sources/datafeed.py	Thu May 31 15:56:21 2012 +0200
@@ -28,7 +28,6 @@
 from cookielib import CookieJar
 
 from lxml import etree
-from logilab.mtconverter import xml_escape
 
 from cubicweb import RegistryNotFound, ObjectNotFound, ValidationError, UnknownEid
 from cubicweb.server.sources import AbstractSource
@@ -68,7 +67,7 @@
           }),
         ('delete-entities',
          {'type' : 'yn',
-          'default': True,
+          'default': False,
           'help': ('Should already imported entities not found anymore on the '
                    'external source be deleted?'),
           'group': 'datafeed-source', 'level': 2,
@@ -80,6 +79,7 @@
           'group': 'datafeed-source', 'level': 2,
           }),
         )
+
     def __init__(self, repo, source_config, eid=None):
         AbstractSource.__init__(self, repo, source_config, eid)
         self.update_config(None, self.check_conf_dict(eid, source_config,
@@ -192,23 +192,13 @@
             self.release_synchronization_lock(session)
 
     def _pull_data(self, session, force=False, raise_on_error=False):
-        if self.config['delete-entities']:
-            myuris = self.source_cwuris(session)
-        else:
-            myuris = None
         importlog = self.init_import_log(session)
+        myuris = self.source_cwuris(session)
         parser = self._get_parser(session, sourceuris=myuris, import_log=importlog)
         if self.process_urls(parser, self.urls, raise_on_error):
             self.warning("some error occured, don't attempt to delete entities")
-        elif self.config['delete-entities'] and myuris:
-            byetype = {}
-            for extid, (eid, etype) in myuris.iteritems():
-                if parser.is_deleted(extid, etype, eid):
-                    byetype.setdefault(etype, []).append(str(eid))
-            for etype, eids in byetype.iteritems():
-                self.warning('delete %s %s entities', len(eids), etype)
-                session.execute('DELETE %s X WHERE X eid IN (%s)'
-                                % (etype, ','.join(eids)))
+        else:
+            parser.handle_deletion(self.config, session, myuris)
         self.update_latest_retrieval(session)
         stats = parser.stats
         if stats.get('created'):
@@ -376,6 +366,17 @@
         """
         return True
 
+    def handle_deletion(self, config, session, myuris):
+        if config['delete-entities'] and myuris:
+            byetype = {}
+            for extid, (eid, etype) in myuris.iteritems():
+                if self.is_deleted(extid, etype, eid):
+                    byetype.setdefault(etype, []).append(str(eid))
+            for etype, eids in byetype.iteritems():
+                self.warning('delete %s %s entities', len(eids), etype)
+                session.execute('DELETE %s X WHERE X eid IN (%s)'
+                                % (etype, ','.join(eids)))
+
     def update_if_necessary(self, entity, attrs):
         self.notify_updated(entity)
         entity.complete(tuple(attrs))