[datafeed] rename parser.sourceuris to source_uris
Its more readable. Also, uniformize related method and local identifier names.
--- a/cubicweb/server/sources/datafeed.py Tue Jun 30 10:05:46 2015 +0200
+++ b/cubicweb/server/sources/datafeed.py Wed Jul 01 08:42:44 2015 +0200
@@ -232,15 +232,16 @@
def _pull_data(self, cnx, force=False, raise_on_error=False, import_log_eid=None):
importlog = self.init_import_log(cnx, import_log_eid)
- myuris = self.source_cwuris(cnx)
+ source_uris = self.source_uris(cnx)
try:
- parser = self._get_parser(cnx, sourceuris=myuris, import_log=importlog)
+ parser = self._get_parser(cnx, import_log=importlog,
+ source_uris=source_uris)
except ObjectNotFound:
return {}
if parser.process_urls(self.urls, raise_on_error):
self.warning("some error occurred, don't attempt to delete entities")
else:
- parser.handle_deletion(self.config, cnx, myuris)
+ parser.handle_deletion(self.config, cnx, source_uris)
self.update_latest_retrieval(cnx)
stats = parser.stats
if stats.get('created'):
@@ -287,7 +288,7 @@
call_hooks('after_add_relation', cnx,
eidfrom=entity.eid, rtype=attr, eidto=value)
- def source_cwuris(self, cnx):
+ def source_uris(self, cnx):
sql = ('SELECT extid, eid, type FROM entities, cw_source_relation '
'WHERE entities.eid=cw_source_relation.eid_from '
'AND cw_source_relation.eid_to=%s' % self.eid)
@@ -310,11 +311,13 @@
class DataFeedParser(AppObject):
__registry__ = 'parsers'
- def __init__(self, cnx, source, sourceuris=None, import_log=None, **kwargs):
+ def __init__(self, cnx, source, import_log=None, source_uris=None, **kwargs):
super(DataFeedParser, self).__init__(cnx, **kwargs)
self.source = source
- self.sourceuris = sourceuris
self.import_log = import_log
+ if source_uris is None:
+ source_uris = {}
+ self.source_uris = source_uris
self.stats = {'created': set(), 'updated': set(), 'checked': set()}
def normalize_url(self, url):
@@ -426,8 +429,8 @@
return None
self.notify_updated(entity) # avoid later update from the source's data
return entity
- if self.sourceuris is not None:
- self.sourceuris.pop(str(uri), None)
+ if self.source_uris is not None:
+ self.source_uris.pop(str(uri), None)
return cnx.entity_from_eid(eid, etype)
def process_urls(self, urls, raise_on_error=False):
@@ -484,10 +487,10 @@
"""
return True
- def handle_deletion(self, config, cnx, myuris):
- if config['delete-entities'] and myuris:
+ def handle_deletion(self, config, cnx, source_uris):
+ if config['delete-entities'] and source_uris:
byetype = {}
- for extid, (eid, etype) in myuris.items():
+ for extid, (eid, etype) in source_uris.items():
if self.is_deleted(extid, etype, eid):
byetype.setdefault(etype, []).append(str(eid))
for etype, eids in byetype.items():
--- a/cubicweb/server/test/unittest_datafeed.py Tue Jun 30 10:05:46 2015 +0200
+++ b/cubicweb/server/test/unittest_datafeed.py Wed Jul 01 08:42:44 2015 +0200
@@ -107,7 +107,7 @@
self.assertEqual(self.repo._extid_cache[b'http://www.cubicweb.org/'],
entity.eid)
- self.assertEqual(dfsource.source_cwuris(cnx),
+ self.assertEqual(dfsource.source_uris(cnx),
{b'http://www.cubicweb.org/': (entity.eid, 'Card')})
self.assertTrue(dfsource.latest_retrieval)
self.assertTrue(dfsource.fresh())
--- a/cubicweb/sobjects/ldapparser.py Tue Jun 30 10:05:46 2015 +0200
+++ b/cubicweb/sobjects/ldapparser.py Wed Jul 01 08:42:44 2015 +0200
@@ -184,10 +184,10 @@
# not found, create it. first forge an external id
extuser.values.setdefault('use_email', []).append(emailextid)
yield importer.ExtEntity('EmailAddress', emailextid, dict(address=[emailaddr]))
- elif self.sourceuris:
- # pop from sourceuris anyway, else email may be removed by the
+ elif self.source_uris:
+ # pop from source_uris anyway, else email may be removed by the
# source once import is finished
- self.sourceuris.pop(emailextid, None)
+ self.source_uris.pop(emailextid, None)
# XXX else check use_email relation?
def handle_deletion(self, config, cnx, myuris):