[datafeed] Complete the import log even if parser could not be found
It happens that if the parser could not be found, _pull_data() would just
return an empty dict without taking care to close the import log which it just
opened. This leads to misleading information in the user interface where
CWDataImport entities kept accumulating in the "imports" tab of CWSource
primary view without anything else happening.
So:
* log an error message when parser cannot be found
* always close (write logs and set "end_timestamp" attribute) import log when
leaving _pull_data().
Closes #15505460.
--- a/cubicweb/server/sources/datafeed.py Fri Oct 21 18:03:06 2016 +0200
+++ b/cubicweb/server/sources/datafeed.py Wed Sep 28 11:06:28 2016 +0200
@@ -238,13 +238,17 @@
source_uris=source_uris,
moved_uris=self.moved_uris(cnx))
except ObjectNotFound:
- return {}
- if parser.process_urls(self.urls, raise_on_error):
- self.warning("some error occurred, don't attempt to delete entities")
+ msg = 'failed to load parser for %s'
+ importlog.record_error(msg % ('source "%s"' % self.uri))
+ self.error(msg, self)
+ stats = {}
else:
- parser.handle_deletion(self.config, cnx, source_uris)
+ if parser.process_urls(self.urls, raise_on_error):
+ self.warning("some error occurred, don't attempt to delete entities")
+ else:
+ parser.handle_deletion(self.config, cnx, source_uris)
+ stats = parser.stats
self.update_latest_retrieval(cnx)
- stats = parser.stats
if stats.get('created'):
importlog.record_info('added %s entities' % len(stats['created']))
if stats.get('updated'):
--- a/cubicweb/server/test/unittest_datafeed.py Fri Oct 21 18:03:06 2016 +0200
+++ b/cubicweb/server/test/unittest_datafeed.py Wed Sep 28 11:06:28 2016 +0200
@@ -52,6 +52,10 @@
yield self.repo.sources_by_uri[u'ô myfeed']._get_parser(session)
else:
yield
+ # vreg.unregister just pops appobjects from their regid entry,
+ # completely remove the entry to ensure we have no side effect with
+ # this empty entry.
+ del self.vreg['parsers'][AParser.__regid__]
def test(self):
self.assertIn(u'ô myfeed', self.repo.sources_by_uri)
@@ -150,6 +154,17 @@
cnx.commit()
self.assertEqual(dfsource.urls, [u"http://pouet.com", u"http://pouet.org"])
+ def test_parser_not_found(self):
+ dfsource = self.repo.sources_by_uri[u'ô myfeed']
+ with self.assertLogs('cubicweb.sources.o myfeed', level='ERROR') as cm:
+ with self.repo.internal_cnx() as cnx:
+ stats = dfsource.pull_data(cnx, force=True)
+ importlog = cnx.find('CWDataImport').one().log
+ self.assertIn('failed to load parser for', cm.output[0])
+ self.assertEqual(stats, {})
+ self.assertIn(u'failed to load parser for source "ô myfeed"',
+ importlog)
+
class DataFeedConfigTC(CubicWebTC):