# HG changeset patch
# User sylvain.thenault@logilab.fr
# Date 1238852757 -7200
# Node ID fa29b5b60107154e55eed4e790368dbc05be7ea3
# Parent  c836bdb3b17be35ab2316846dc09a03f38e65de9
set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query

diff -r c836bdb3b17b -r fa29b5b60107 server/sources/__init__.py
--- a/server/sources/__init__.py	Sat Apr 04 15:16:37 2009 +0200
+++ b/server/sources/__init__.py	Sat Apr 04 15:45:57 2009 +0200
@@ -1,15 +1,35 @@
 """cubicweb server sources support
 
 :organization: Logilab
-:copyright: 2001-2008 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+:copyright: 2001-2009 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
 :contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr
 """
 __docformat__ = "restructuredtext en"
 
 from logging import getLogger
 
+from mx.DateTime import now, DateTimeDelta
+
 from cubicweb import set_log_methods
 
+class TimedCache(dict):
+    def __init__(self, ttlm, ttls=0):
+        # time to live in minutes
+        self.ttl = DateTimeDelta(0, 0, ttlm, ttls)
+        
+    def __setitem__(self, key, value):
+        dict.__setitem__(self, key, (now(), value))
+        
+    def __getitem__(self, key):
+        return dict.__getitem__(self, key)[1]
+    
+    def clear_expired(self):
+        now_ = now()
+        ttl = self.ttl
+        for key, (timestamp, value) in self.items():
+            if now_ - timestamp > ttl:
+                del self[key]
+
 
 class AbstractSource(object):
     """an abstract class for sources"""
diff -r c836bdb3b17b -r fa29b5b60107 server/sources/ldapuser.py
--- a/server/sources/ldapuser.py	Sat Apr 04 15:16:37 2009 +0200
+++ b/server/sources/ldapuser.py	Sat Apr 04 15:45:57 2009 +0200
@@ -3,7 +3,7 @@
 this source is for now limited to a read-only EUser source
 
 :organization: Logilab
-:copyright: 2003-2008 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+:copyright: 2003-2009 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
 :contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr
 
 
@@ -20,8 +20,6 @@
 FOR A PARTICULAR PURPOSE.
 """
 
-from mx.DateTime import now, DateTimeDelta
-
 from logilab.common.textutils import get_csv
 from rql.nodes import Relation, VariableRef, Constant, Function
 
@@ -30,9 +28,10 @@
 from ldap.filter import filter_format, escape_filter_chars
 from ldapurl import LDAPUrl
 
-from cubicweb.common import AuthenticationError, UnknownEid, RepositoryError
-from cubicweb.server.sources import AbstractSource, TrFunc, GlobTrFunc, ConnectionWrapper
+from cubicweb import AuthenticationError, UnknownEid, RepositoryError
 from cubicweb.server.utils import cartesian_product
+from cubicweb.server.sources import (AbstractSource, TrFunc, GlobTrFunc,
+                                     ConnectionWrapper, TimedCache)
 
 # search scopes
 BASE = ldap.SCOPE_BASE
@@ -49,24 +48,6 @@
     1: (636, 'ldaps'),
     2: (0,   'ldapi'),
     }
-
-class TimedCache(dict):
-    def __init__(self, ttlm, ttls=0):
-        # time to live in minutes
-        self.ttl = DateTimeDelta(0, 0, ttlm, ttls)
-        
-    def __setitem__(self, key, value):
-        dict.__setitem__(self, key, (now(), value))
-        
-    def __getitem__(self, key):
-        return dict.__getitem__(self, key)[1]
-    
-    def clear_expired(self):
-        now_ = now()
-        ttl = self.ttl
-        for key, (timestamp, value) in self.items():
-            if now_ - timestamp > ttl:
-                del self[key]
                 
 class LDAPUserSource(AbstractSource):
     """LDAP read-only EUser source"""
diff -r c836bdb3b17b -r fa29b5b60107 server/sources/pyrorql.py
--- a/server/sources/pyrorql.py	Sat Apr 04 15:16:37 2009 +0200
+++ b/server/sources/pyrorql.py	Sat Apr 04 15:45:57 2009 +0200
@@ -1,7 +1,7 @@
 """Source to query another RQL repository using pyro
 
 :organization: Logilab
-:copyright: 2007-2008 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+:copyright: 2007-2009 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
 :contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr
 """
 __docformat__ = "restructuredtext en"
@@ -21,7 +21,7 @@
 from cubicweb import dbapi, server
 from cubicweb import BadConnectionId, UnknownEid, ConnectionError
 from cubicweb.cwconfig import register_persistent_options
-from cubicweb.server.sources import AbstractSource, ConnectionWrapper
+from cubicweb.server.sources import AbstractSource, ConnectionWrapper, TimedCache
 
 class ReplaceByInOperator:
     def __init__(self, eids):
@@ -129,6 +129,7 @@
                        'group': 'sources', 
                        }),)
         register_persistent_options(myoptions)
+        self._query_cache = TimedCache(30)
 
     def last_update_time(self):
         pkey = u'sources.%s.latest-update-time' % self.uri
@@ -153,6 +154,7 @@
         """method called by the repository once ready to handle request"""
         interval = int(self.config.get('synchronization-interval', 5*60))
         self.repo.looping_task(interval, self.synchronize) 
+        self.repo.looping_task(self._query_cache.ttl.seconds/10, self._query_cache.clear_expired) 
 
     def synchronize(self, mtime=None):
         """synchronize content known by this repository with content in the
@@ -240,9 +242,20 @@
         # try to reconnect
         return self.get_connection()
         
-    
     def syntax_tree_search(self, session, union, args=None, cachekey=None,
                            varmap=None):
+        assert not varmap, (varmap, union)
+        rqlkey = union.as_string(kwargs=args)
+        try:
+            results = self._query_cache[rqlkey]
+            print 'cache hit', rqlkey
+        except KeyError:
+            results = self._syntax_tree_search(session, union, args)
+            print 'cache miss', rqlkey
+            self._query_cache[rqlkey] = results
+        return results
+    
+    def _syntax_tree_search(self, session, union, args):
         """return result from this source for a rql query (actually from a rql 
         syntax tree and a solution dictionary mapping each used variable to a 
         possible type). If cachekey is given, the query necessary to fetch the