# HG changeset patch # User Sylvain Thénault # Date 1297271178 -3600 # Node ID 861251f125cfb9f3ed05d6e57c5b4d1075682938 # Parent ffda12be2e9fe5c6f488725a37d66f7b75989049 [pyro source] benefit from addition of latest_retrieval on CWSource to avoid using a CWProperty to store pyrorql source latest synchronization. diff -r ffda12be2e9f -r 861251f125cf misc/migration/3.11.0_Any.py --- a/misc/migration/3.11.0_Any.py Wed Feb 09 18:06:17 2011 +0100 +++ b/misc/migration/3.11.0_Any.py Wed Feb 09 18:06:18 2011 +0100 @@ -1,3 +1,5 @@ +from datetime import datetime + for rtype in ('cw_support', 'cw_dont_cross', 'cw_may_cross'): drop_relation_type(rtype) @@ -73,3 +75,10 @@ cw_for_source=source, cw_schema=session.entity_from_eid(schema[etype].eid), options=u'dontcross') + # latest update time cwproperty is now a source attribute (latest_retrieval) + pkey = u'sources.%s.latest-update-time' % source.uri + rset = session.execute('Any V WHERE X is CWProperty, X value V, X pkey %(k)s', + {'k': pkey}) + timestamp = int(rset[0][0]) + sourceentity.set_attributes(latest_retrieval=datetime.fromtimestamp(timestamp)) + session.execute('DELETE CWProperty X WHERE X pkey %(k)s', {'k': pkey}) diff -r ffda12be2e9f -r 861251f125cf server/sources/pyrorql.py --- a/server/sources/pyrorql.py Wed Feb 09 18:06:17 2011 +0100 +++ b/server/sources/pyrorql.py Wed Feb 09 18:06:18 2011 +0100 @@ -127,13 +127,6 @@ AbstractSource.__init__(self, repo, source_config, eid) self.update_config(None, self.check_conf_dict(eid, source_config, fail_if_unknown=False)) - myoptions = (('%s.latest-update-time' % self.uri, - {'type' : 'int', 'sitewide': True, - 'default': 0, - 'help': _('timestamp of the latest source synchronization.'), - 'group': 'sources', - }),) - register_persistent_options(myoptions) self._query_cache = TimedCache(1800) def update_config(self, source_entity, processed_config): @@ -144,30 +137,13 @@ processed_config['base-url'] += '/' self.config = processed_config self._skip_externals = processed_config['skip-external-entities'] + if source_entity is not None: + self.latest_retrieval = source_entity.latest_retrieval def reset_caches(self): """method called during test to reset potential source caches""" self._query_cache = TimedCache(1800) - def last_update_time(self): - pkey = u'sources.%s.latest-update-time' % self.uri - session = self.repo.internal_session() - try: - rset = session.execute('Any V WHERE X is CWProperty, X value V, X pkey %(k)s', - {'k': pkey}) - if not rset: - # insert it - session.execute('INSERT CWProperty X: X pkey %(k)s, X value %(v)s', - {'k': pkey, 'v': u'0'}) - session.commit() - timestamp = 0 - else: - assert len(rset) == 1 - timestamp = int(rset[0][0]) - return datetime.fromtimestamp(timestamp) - finally: - session.close() - def init(self, activated, source_entity): """method called by the repository once ready to handle request""" self.load_mapping(source_entity._cw) @@ -176,6 +152,7 @@ self.repo.looping_task(interval, self.synchronize) self.repo.looping_task(self._query_cache.ttl.seconds/10, self._query_cache.clear_expired) + self.latest_retrieval = source_entity.latest_retrieval def load_mapping(self, session=None): self.support_entities = {} @@ -279,7 +256,7 @@ return etypes = self.support_entities.keys() if mtime is None: - mtime = self.last_update_time() + mtime = self.latest_retrieval updatetime, modified, deleted = extrepo.entities_modified_since( etypes, mtime) self._query_cache.clear() @@ -312,9 +289,9 @@ self.exception('while updating %s with external id %s of source %s', etype, extid, self.uri) continue - session.execute('SET X value %(v)s WHERE X pkey %(k)s', - {'k': u'sources.%s.latest-update-time' % self.uri, - 'v': unicode(int(mktime(updatetime.timetuple())))}) + self.latest_retrieval = updatetime + session.execute('SET X latest_retrieval %(date)s WHERE X eid %(x)s', + {'x': self.eid, 'date': self.latest_retrieval}) session.commit() finally: session.close()