set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
--- a/server/sources/__init__.py Sat Apr 04 15:16:37 2009 +0200
+++ b/server/sources/__init__.py Sat Apr 04 15:45:57 2009 +0200
@@ -1,15 +1,35 @@
"""cubicweb server sources support
:organization: Logilab
-:copyright: 2001-2008 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+:copyright: 2001-2009 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr
"""
__docformat__ = "restructuredtext en"
from logging import getLogger
+from mx.DateTime import now, DateTimeDelta
+
from cubicweb import set_log_methods
+class TimedCache(dict):
+ def __init__(self, ttlm, ttls=0):
+ # time to live in minutes
+ self.ttl = DateTimeDelta(0, 0, ttlm, ttls)
+
+ def __setitem__(self, key, value):
+ dict.__setitem__(self, key, (now(), value))
+
+ def __getitem__(self, key):
+ return dict.__getitem__(self, key)[1]
+
+ def clear_expired(self):
+ now_ = now()
+ ttl = self.ttl
+ for key, (timestamp, value) in self.items():
+ if now_ - timestamp > ttl:
+ del self[key]
+
class AbstractSource(object):
"""an abstract class for sources"""
--- a/server/sources/ldapuser.py Sat Apr 04 15:16:37 2009 +0200
+++ b/server/sources/ldapuser.py Sat Apr 04 15:45:57 2009 +0200
@@ -3,7 +3,7 @@
this source is for now limited to a read-only EUser source
:organization: Logilab
-:copyright: 2003-2008 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+:copyright: 2003-2009 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr
@@ -20,8 +20,6 @@
FOR A PARTICULAR PURPOSE.
"""
-from mx.DateTime import now, DateTimeDelta
-
from logilab.common.textutils import get_csv
from rql.nodes import Relation, VariableRef, Constant, Function
@@ -30,9 +28,10 @@
from ldap.filter import filter_format, escape_filter_chars
from ldapurl import LDAPUrl
-from cubicweb.common import AuthenticationError, UnknownEid, RepositoryError
-from cubicweb.server.sources import AbstractSource, TrFunc, GlobTrFunc, ConnectionWrapper
+from cubicweb import AuthenticationError, UnknownEid, RepositoryError
from cubicweb.server.utils import cartesian_product
+from cubicweb.server.sources import (AbstractSource, TrFunc, GlobTrFunc,
+ ConnectionWrapper, TimedCache)
# search scopes
BASE = ldap.SCOPE_BASE
@@ -49,24 +48,6 @@
1: (636, 'ldaps'),
2: (0, 'ldapi'),
}
-
-class TimedCache(dict):
- def __init__(self, ttlm, ttls=0):
- # time to live in minutes
- self.ttl = DateTimeDelta(0, 0, ttlm, ttls)
-
- def __setitem__(self, key, value):
- dict.__setitem__(self, key, (now(), value))
-
- def __getitem__(self, key):
- return dict.__getitem__(self, key)[1]
-
- def clear_expired(self):
- now_ = now()
- ttl = self.ttl
- for key, (timestamp, value) in self.items():
- if now_ - timestamp > ttl:
- del self[key]
class LDAPUserSource(AbstractSource):
"""LDAP read-only EUser source"""
--- a/server/sources/pyrorql.py Sat Apr 04 15:16:37 2009 +0200
+++ b/server/sources/pyrorql.py Sat Apr 04 15:45:57 2009 +0200
@@ -1,7 +1,7 @@
"""Source to query another RQL repository using pyro
:organization: Logilab
-:copyright: 2007-2008 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+:copyright: 2007-2009 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr
"""
__docformat__ = "restructuredtext en"
@@ -21,7 +21,7 @@
from cubicweb import dbapi, server
from cubicweb import BadConnectionId, UnknownEid, ConnectionError
from cubicweb.cwconfig import register_persistent_options
-from cubicweb.server.sources import AbstractSource, ConnectionWrapper
+from cubicweb.server.sources import AbstractSource, ConnectionWrapper, TimedCache
class ReplaceByInOperator:
def __init__(self, eids):
@@ -129,6 +129,7 @@
'group': 'sources',
}),)
register_persistent_options(myoptions)
+ self._query_cache = TimedCache(30)
def last_update_time(self):
pkey = u'sources.%s.latest-update-time' % self.uri
@@ -153,6 +154,7 @@
"""method called by the repository once ready to handle request"""
interval = int(self.config.get('synchronization-interval', 5*60))
self.repo.looping_task(interval, self.synchronize)
+ self.repo.looping_task(self._query_cache.ttl.seconds/10, self._query_cache.clear_expired)
def synchronize(self, mtime=None):
"""synchronize content known by this repository with content in the
@@ -240,9 +242,20 @@
# try to reconnect
return self.get_connection()
-
def syntax_tree_search(self, session, union, args=None, cachekey=None,
varmap=None):
+ assert not varmap, (varmap, union)
+ rqlkey = union.as_string(kwargs=args)
+ try:
+ results = self._query_cache[rqlkey]
+ print 'cache hit', rqlkey
+ except KeyError:
+ results = self._syntax_tree_search(session, union, args)
+ print 'cache miss', rqlkey
+ self._query_cache[rqlkey] = results
+ return results
+
+ def _syntax_tree_search(self, session, union, args):
"""return result from this source for a rql query (actually from a rql
syntax tree and a solution dictionary mapping each used variable to a
possible type). If cachekey is given, the query necessary to fetch the