[datafeed] move process_urls to the parser
authorSylvain Thénault <sylvain.thenault@logilab.fr>
Tue, 30 Jun 2015 10:05:46 +0200
changeset 11251 b66a8c3eebeb
parent 11250 597f02c5cf5a
child 11252 6b1d09ef0c45
[datafeed] move process_urls to the parser
cubicweb/server/sources/datafeed.py
cubicweb/sobjects/test/unittest_cwxmlparser.py
--- a/cubicweb/server/sources/datafeed.py	Thu Jan 14 17:14:16 2016 +0100
+++ b/cubicweb/server/sources/datafeed.py	Tue Jun 30 10:05:46 2015 +0200
@@ -237,7 +237,7 @@
             parser = self._get_parser(cnx, sourceuris=myuris, import_log=importlog)
         except ObjectNotFound:
             return {}
-        if self.process_urls(parser, self.urls, raise_on_error):
+        if parser.process_urls(self.urls, raise_on_error):
             self.warning("some error occurred, don't attempt to delete entities")
         else:
             parser.handle_deletion(self.config, cnx, myuris)
@@ -251,28 +251,6 @@
         cnx.commit()
         return stats
 
-    def process_urls(self, parser, urls, raise_on_error=False):
-        error = False
-        for url in urls:
-            self.info('pulling data from %s', url)
-            try:
-                if parser.process(url, raise_on_error):
-                    error = True
-            except IOError as exc:
-                if raise_on_error:
-                    raise
-                parser.import_log.record_error(
-                    'could not pull data while processing %s: %s'
-                    % (url, exc))
-                error = True
-            except Exception as exc:
-                if raise_on_error:
-                    raise
-                self.exception('error while processing %s: %s',
-                               url, exc)
-                error = True
-        return error
-
     @deprecated('[3.21] use the new store API')
     def before_entity_insertion(self, cnx, lid, etype, eid, sourceparams):
         """called by the repository when an eid has been attributed for an
@@ -446,12 +424,35 @@
                 entity = cnx.entity_from_eid(-eid)
             except UnknownEid:
                 return None
-            self.notify_updated(entity) # avoid later update from the source's data
+            self.notify_updated(entity)  # avoid later update from the source's data
             return entity
         if self.sourceuris is not None:
             self.sourceuris.pop(str(uri), None)
         return cnx.entity_from_eid(eid, etype)
 
+    def process_urls(self, urls, raise_on_error=False):
+        error = False
+        for url in urls:
+            self.info('pulling data from %s', url)
+            try:
+                if self.process(url, raise_on_error):
+                    error = True
+            except IOError as exc:
+                if raise_on_error:
+                    raise
+                self.import_log.record_error(
+                    'could not pull data while processing %s: %s'
+                    % (url, exc))
+                error = True
+            except Exception as exc:
+                if raise_on_error:
+                    raise
+                self.import_log.record_error(str(exc))
+                self.exception('error while processing %s: %s',
+                               url, exc)
+                error = True
+        return error
+
     def process(self, url, raise_on_error=False):
         """main callback: process the url"""
         raise NotImplementedError
--- a/cubicweb/sobjects/test/unittest_cwxmlparser.py	Thu Jan 14 17:14:16 2016 +0100
+++ b/cubicweb/sobjects/test/unittest_cwxmlparser.py	Tue Jun 30 10:05:46 2015 +0200
@@ -310,7 +310,7 @@
         dfsource = self.repo.sources_by_uri['myfeed']
         with self.repo.internal_cnx() as cnx:
             parser = dfsource._get_parser(cnx)
-            dfsource.process_urls(parser, ['''
+            parser.process_urls(['''
 <rset size="1">
  <Card eid="50" cwuri="http://pouet.org/50" cwsource="system">
   <title>how-to</title>
@@ -322,7 +322,7 @@
         dfsource = self.repo.sources_by_uri['myfeed']
         with self.repo.internal_cnx() as cnx:
             parser = dfsource._get_parser(cnx)
-            dfsource.process_urls(parser, ['''
+            parser.process_urls(['''
 <rset size="1">
  <Card eid="50" cwuri="http://pouet.org/50" cwsource="system">
   <title>how-to</title>