--- a/cubicweb/server/sources/datafeed.py Thu Jan 14 17:14:16 2016 +0100
+++ b/cubicweb/server/sources/datafeed.py Tue Jun 30 10:05:46 2015 +0200
@@ -237,7 +237,7 @@
parser = self._get_parser(cnx, sourceuris=myuris, import_log=importlog)
except ObjectNotFound:
return {}
- if self.process_urls(parser, self.urls, raise_on_error):
+ if parser.process_urls(self.urls, raise_on_error):
self.warning("some error occurred, don't attempt to delete entities")
else:
parser.handle_deletion(self.config, cnx, myuris)
@@ -251,28 +251,6 @@
cnx.commit()
return stats
- def process_urls(self, parser, urls, raise_on_error=False):
- error = False
- for url in urls:
- self.info('pulling data from %s', url)
- try:
- if parser.process(url, raise_on_error):
- error = True
- except IOError as exc:
- if raise_on_error:
- raise
- parser.import_log.record_error(
- 'could not pull data while processing %s: %s'
- % (url, exc))
- error = True
- except Exception as exc:
- if raise_on_error:
- raise
- self.exception('error while processing %s: %s',
- url, exc)
- error = True
- return error
-
@deprecated('[3.21] use the new store API')
def before_entity_insertion(self, cnx, lid, etype, eid, sourceparams):
"""called by the repository when an eid has been attributed for an
@@ -446,12 +424,35 @@
entity = cnx.entity_from_eid(-eid)
except UnknownEid:
return None
- self.notify_updated(entity) # avoid later update from the source's data
+ self.notify_updated(entity) # avoid later update from the source's data
return entity
if self.sourceuris is not None:
self.sourceuris.pop(str(uri), None)
return cnx.entity_from_eid(eid, etype)
+ def process_urls(self, urls, raise_on_error=False):
+ error = False
+ for url in urls:
+ self.info('pulling data from %s', url)
+ try:
+ if self.process(url, raise_on_error):
+ error = True
+ except IOError as exc:
+ if raise_on_error:
+ raise
+ self.import_log.record_error(
+ 'could not pull data while processing %s: %s'
+ % (url, exc))
+ error = True
+ except Exception as exc:
+ if raise_on_error:
+ raise
+ self.import_log.record_error(str(exc))
+ self.exception('error while processing %s: %s',
+ url, exc)
+ error = True
+ return error
+
def process(self, url, raise_on_error=False):
"""main callback: process the url"""
raise NotImplementedError
--- a/cubicweb/sobjects/test/unittest_cwxmlparser.py Thu Jan 14 17:14:16 2016 +0100
+++ b/cubicweb/sobjects/test/unittest_cwxmlparser.py Tue Jun 30 10:05:46 2015 +0200
@@ -310,7 +310,7 @@
dfsource = self.repo.sources_by_uri['myfeed']
with self.repo.internal_cnx() as cnx:
parser = dfsource._get_parser(cnx)
- dfsource.process_urls(parser, ['''
+ parser.process_urls(['''
<rset size="1">
<Card eid="50" cwuri="http://pouet.org/50" cwsource="system">
<title>how-to</title>
@@ -322,7 +322,7 @@
dfsource = self.repo.sources_by_uri['myfeed']
with self.repo.internal_cnx() as cnx:
parser = dfsource._get_parser(cnx)
- dfsource.process_urls(parser, ['''
+ parser.process_urls(['''
<rset size="1">
<Card eid="50" cwuri="http://pouet.org/50" cwsource="system">
<title>how-to</title>