[pyro source] benefit from addition of latest_retrieval on CWSource
to avoid using a CWProperty to store pyrorql source latest synchronization.
--- a/misc/migration/3.11.0_Any.py Wed Feb 09 18:06:17 2011 +0100
+++ b/misc/migration/3.11.0_Any.py Wed Feb 09 18:06:18 2011 +0100
@@ -1,3 +1,5 @@
+from datetime import datetime
+
for rtype in ('cw_support', 'cw_dont_cross', 'cw_may_cross'):
drop_relation_type(rtype)
@@ -73,3 +75,10 @@
cw_for_source=source,
cw_schema=session.entity_from_eid(schema[etype].eid),
options=u'dontcross')
+ # latest update time cwproperty is now a source attribute (latest_retrieval)
+ pkey = u'sources.%s.latest-update-time' % source.uri
+ rset = session.execute('Any V WHERE X is CWProperty, X value V, X pkey %(k)s',
+ {'k': pkey})
+ timestamp = int(rset[0][0])
+ sourceentity.set_attributes(latest_retrieval=datetime.fromtimestamp(timestamp))
+ session.execute('DELETE CWProperty X WHERE X pkey %(k)s', {'k': pkey})
--- a/server/sources/pyrorql.py Wed Feb 09 18:06:17 2011 +0100
+++ b/server/sources/pyrorql.py Wed Feb 09 18:06:18 2011 +0100
@@ -127,13 +127,6 @@
AbstractSource.__init__(self, repo, source_config, eid)
self.update_config(None, self.check_conf_dict(eid, source_config,
fail_if_unknown=False))
- myoptions = (('%s.latest-update-time' % self.uri,
- {'type' : 'int', 'sitewide': True,
- 'default': 0,
- 'help': _('timestamp of the latest source synchronization.'),
- 'group': 'sources',
- }),)
- register_persistent_options(myoptions)
self._query_cache = TimedCache(1800)
def update_config(self, source_entity, processed_config):
@@ -144,30 +137,13 @@
processed_config['base-url'] += '/'
self.config = processed_config
self._skip_externals = processed_config['skip-external-entities']
+ if source_entity is not None:
+ self.latest_retrieval = source_entity.latest_retrieval
def reset_caches(self):
"""method called during test to reset potential source caches"""
self._query_cache = TimedCache(1800)
- def last_update_time(self):
- pkey = u'sources.%s.latest-update-time' % self.uri
- session = self.repo.internal_session()
- try:
- rset = session.execute('Any V WHERE X is CWProperty, X value V, X pkey %(k)s',
- {'k': pkey})
- if not rset:
- # insert it
- session.execute('INSERT CWProperty X: X pkey %(k)s, X value %(v)s',
- {'k': pkey, 'v': u'0'})
- session.commit()
- timestamp = 0
- else:
- assert len(rset) == 1
- timestamp = int(rset[0][0])
- return datetime.fromtimestamp(timestamp)
- finally:
- session.close()
-
def init(self, activated, source_entity):
"""method called by the repository once ready to handle request"""
self.load_mapping(source_entity._cw)
@@ -176,6 +152,7 @@
self.repo.looping_task(interval, self.synchronize)
self.repo.looping_task(self._query_cache.ttl.seconds/10,
self._query_cache.clear_expired)
+ self.latest_retrieval = source_entity.latest_retrieval
def load_mapping(self, session=None):
self.support_entities = {}
@@ -279,7 +256,7 @@
return
etypes = self.support_entities.keys()
if mtime is None:
- mtime = self.last_update_time()
+ mtime = self.latest_retrieval
updatetime, modified, deleted = extrepo.entities_modified_since(
etypes, mtime)
self._query_cache.clear()
@@ -312,9 +289,9 @@
self.exception('while updating %s with external id %s of source %s',
etype, extid, self.uri)
continue
- session.execute('SET X value %(v)s WHERE X pkey %(k)s',
- {'k': u'sources.%s.latest-update-time' % self.uri,
- 'v': unicode(int(mktime(updatetime.timetuple())))})
+ self.latest_retrieval = updatetime
+ session.execute('SET X latest_retrieval %(date)s WHERE X eid %(x)s',
+ {'x': self.eid, 'date': self.latest_retrieval})
session.commit()
finally:
session.close()