[pyro source] benefit from addition of latest_retrieval on CWSource
authorSylvain Thénault <sylvain.thenault@logilab.fr>
Wed, 09 Feb 2011 18:06:18 +0100
changeset 6958 861251f125cf
parent 6957 ffda12be2e9f
child 6959 037a0277db0a
[pyro source] benefit from addition of latest_retrieval on CWSource to avoid using a CWProperty to store pyrorql source latest synchronization.
misc/migration/3.11.0_Any.py
server/sources/pyrorql.py
--- a/misc/migration/3.11.0_Any.py	Wed Feb 09 18:06:17 2011 +0100
+++ b/misc/migration/3.11.0_Any.py	Wed Feb 09 18:06:18 2011 +0100
@@ -1,3 +1,5 @@
+from datetime import datetime
+
 for rtype in ('cw_support', 'cw_dont_cross', 'cw_may_cross'):
     drop_relation_type(rtype)
 
@@ -73,3 +75,10 @@
                           cw_for_source=source,
                           cw_schema=session.entity_from_eid(schema[etype].eid),
                           options=u'dontcross')
+        # latest update time cwproperty is now a source attribute (latest_retrieval)
+        pkey = u'sources.%s.latest-update-time' % source.uri
+        rset = session.execute('Any V WHERE X is CWProperty, X value V, X pkey %(k)s',
+                               {'k': pkey})
+        timestamp = int(rset[0][0])
+        sourceentity.set_attributes(latest_retrieval=datetime.fromtimestamp(timestamp))
+        session.execute('DELETE CWProperty X WHERE X pkey %(k)s', {'k': pkey})
--- a/server/sources/pyrorql.py	Wed Feb 09 18:06:17 2011 +0100
+++ b/server/sources/pyrorql.py	Wed Feb 09 18:06:18 2011 +0100
@@ -127,13 +127,6 @@
         AbstractSource.__init__(self, repo, source_config, eid)
         self.update_config(None, self.check_conf_dict(eid, source_config,
                                                       fail_if_unknown=False))
-        myoptions = (('%s.latest-update-time' % self.uri,
-                      {'type' : 'int', 'sitewide': True,
-                       'default': 0,
-                       'help': _('timestamp of the latest source synchronization.'),
-                       'group': 'sources',
-                       }),)
-        register_persistent_options(myoptions)
         self._query_cache = TimedCache(1800)
 
     def update_config(self, source_entity, processed_config):
@@ -144,30 +137,13 @@
             processed_config['base-url'] += '/'
         self.config = processed_config
         self._skip_externals = processed_config['skip-external-entities']
+        if source_entity is not None:
+            self.latest_retrieval = source_entity.latest_retrieval
 
     def reset_caches(self):
         """method called during test to reset potential source caches"""
         self._query_cache = TimedCache(1800)
 
-    def last_update_time(self):
-        pkey = u'sources.%s.latest-update-time' % self.uri
-        session = self.repo.internal_session()
-        try:
-            rset = session.execute('Any V WHERE X is CWProperty, X value V, X pkey %(k)s',
-                                   {'k': pkey})
-            if not rset:
-                # insert it
-                session.execute('INSERT CWProperty X: X pkey %(k)s, X value %(v)s',
-                                {'k': pkey, 'v': u'0'})
-                session.commit()
-                timestamp = 0
-            else:
-                assert len(rset) == 1
-                timestamp = int(rset[0][0])
-            return datetime.fromtimestamp(timestamp)
-        finally:
-            session.close()
-
     def init(self, activated, source_entity):
         """method called by the repository once ready to handle request"""
         self.load_mapping(source_entity._cw)
@@ -176,6 +152,7 @@
             self.repo.looping_task(interval, self.synchronize)
             self.repo.looping_task(self._query_cache.ttl.seconds/10,
                                    self._query_cache.clear_expired)
+            self.latest_retrieval = source_entity.latest_retrieval
 
     def load_mapping(self, session=None):
         self.support_entities = {}
@@ -279,7 +256,7 @@
             return
         etypes = self.support_entities.keys()
         if mtime is None:
-            mtime = self.last_update_time()
+            mtime = self.latest_retrieval
         updatetime, modified, deleted = extrepo.entities_modified_since(
             etypes, mtime)
         self._query_cache.clear()
@@ -312,9 +289,9 @@
                     self.exception('while updating %s with external id %s of source %s',
                                    etype, extid, self.uri)
                     continue
-            session.execute('SET X value %(v)s WHERE X pkey %(k)s',
-                            {'k': u'sources.%s.latest-update-time' % self.uri,
-                             'v': unicode(int(mktime(updatetime.timetuple())))})
+            self.latest_retrieval = updatetime
+            session.execute('SET X latest_retrieval %(date)s WHERE X eid %(x)s',
+                            {'x': self.eid, 'date': self.latest_retrieval})
             session.commit()
         finally:
             session.close()