# HG changeset patch # User Sylvain Thénault # Date 1249136020 -7200 # Node ID d6012db7b93ee6c08cf87de206517f200acc44aa # Parent c0b5b189190e3372b523361191b23211d4a4d7f6 R [server debug] more server side debugging tweaks diff -r c0b5b189190e -r d6012db7b93e server/sources/__init__.py --- a/server/sources/__init__.py Sat Aug 01 01:24:42 2009 +0200 +++ b/server/sources/__init__.py Sat Aug 01 16:13:40 2009 +0200 @@ -11,11 +11,32 @@ from datetime import datetime, timedelta from logging import getLogger -from cubicweb import set_log_methods +from cubicweb import set_log_methods, server from cubicweb.schema import VIRTUAL_RTYPES from cubicweb.server.sqlutils import SQL_PREFIX +def dbg_st_search(uri, union, varmap, args, cachekey=None, prefix='rql for'): + if server.DEBUG & server.DBG_RQL: + print '%s %s source: %s' % (prefix, uri, union.as_string()) + if varmap: + print ' using varmap', varmap + if server.DEBUG & server.DBG_MORE: + print ' args', args + print ' cache key', cachekey + print ' solutions', ','.join(str(s.solutions) + for s in union.children) + # return true so it can be used as assertion (and so be killed by python -O) + return True + +def dbg_results(results): + if server.DEBUG & server.DBG_RQL: + if len(results) > 10: + print ' -->', results[:10], '...', len(results) + else: + print ' -->', results + # return true so it can be used as assertion (and so be killed by python -O) + return True class TimedCache(dict): def __init__(self, ttlm, ttls=0): diff -r c0b5b189190e -r d6012db7b93e server/sources/extlite.py --- a/server/sources/extlite.py Sat Aug 01 01:24:42 2009 +0200 +++ b/server/sources/extlite.py Sat Aug 01 16:13:40 2009 +0200 @@ -13,8 +13,8 @@ from cubicweb import server from cubicweb.server.sqlutils import (SQL_PREFIX, SQLAdapterMixIn, sqlexec, sql_source_backup, sql_source_restore) -from cubicweb.server.sources import AbstractSource, native -from cubicweb.server.sources.rql2sql import SQLGenerator +from cubicweb.server.sources import native, rql2sql +from cubicweb.server.sources import AbstractSource, dbg_st_search, dbg_results class ConnectionWrapper(object): def __init__(self, source=None): @@ -31,32 +31,33 @@ if self._cnx is None: self._cnx = self.source._sqlcnx if server.DEBUG & server.DBG_SQL: - print 'sql cnx open', self._cnx + print 'sql cnx OPEN', self._cnx return self._cnx.cursor() def commit(self): if self._cnx is not None: if server.DEBUG & server.DBG_SQL: - print 'sql cnx commit', self._cnx + print 'sql cnx COMMIT', self._cnx + self._cnx.commit() def rollback(self): if self._cnx is not None: if server.DEBUG & server.DBG_SQL: - print 'sql cnx rollback', self._cnx + print 'sql cnx ROLLBACK', self._cnx self._cnx.rollback() def close(self): if self._cnx is not None: + if server.DEBUG & server.DBG_SQL: + print 'sql cnx CLOSE', self._cnx self._cnx.close() - if server.DEBUG & server.DBG_SQL: - print 'sql cnx close', self._cnx self._cnx = None class SQLiteAbstractSource(AbstractSource): """an abstract class for external sources using a sqlite database helper """ - sqlgen_class = SQLGenerator + sqlgen_class = rql2sql.SQLGenerator @classmethod def set_nonsystem_types(cls): # those entities are only in this source, we don't want them in the @@ -180,8 +181,8 @@ # connection cnx.close() - def syntax_tree_search(self, session, union, - args=None, cachekey=None, varmap=None, debug=0): + def syntax_tree_search(self, session, union, args=None, cachekey=None, + varmap=None): """return result from this source for a rql query (actually from a rql syntax tree and a solution dictionary mapping each used variable to a possible type). If cachekey is given, the query necessary to fetch the @@ -189,14 +190,12 @@ """ if self._need_sql_create: return [] + assert dbg_st_search(self.uri, union, varmap, args, cachekey) sql, query_args = self.rqlsqlgen.generate(union, args) - if server.DEBUG & server.DBG_RQL: - print 'RQL FOR %s SOURCE:' % (self.uri, union.as_string()) args = self.sqladapter.merge_args(args, query_args) - res = self.sqladapter.process_result(self.doexec(session, sql, args)) - if server.DEBUG & (server.DBG_SQL | server.DBG_RQL): - print '------>', res - return res + results = self.sqladapter.process_result(self.doexec(session, sql, args)) + assert dbg_results(results) + return results def local_add_entity(self, session, entity): """insert the entity in the local database. diff -r c0b5b189190e -r d6012db7b93e server/sources/native.py --- a/server/sources/native.py Sat Aug 01 01:24:42 2009 +0200 +++ b/server/sources/native.py Sat Aug 01 16:13:40 2009 +0200 @@ -28,7 +28,7 @@ from cubicweb.server.sqlutils import (SQL_PREFIX, SQLAdapterMixIn, sql_source_backup, sql_source_restore) from cubicweb.server.rqlannotation import set_qdata -from cubicweb.server.sources import AbstractSource +from cubicweb.server.sources import AbstractSource, dbg_st_search, dbg_results from cubicweb.server.sources.rql2sql import SQLGenerator @@ -59,6 +59,7 @@ def fetchone(self): return self.cu.fetchone() + def make_schema(selected, solution, table, typemap): """return a sql schema to store RQL query result""" sql = [] @@ -75,6 +76,7 @@ sql.append('%s %s' % (name, typemap['Int'])) return ','.join(sql), varmap + def _modified_sql(table, etypes): # XXX protect against sql injection if len(etypes) > 1: @@ -304,14 +306,7 @@ necessary to fetch the results (but not the results themselves) may be cached using this key. """ - if server.DEBUG & server.DBG_RQL: - print 'RQL FOR NATIVE SOURCE %s: %s' % (self.uri, union.as_string()) - if varmap: - print 'using varmap', varmap - if server.DEBUG & server.DBG_MORE: - print 'args', args - print 'cache key', cachekey - print 'solutions', ','.join(str(s.solutions) for s in union.children) + assert dbg_st_search(self.uri, union, varmap, args, cachekey) # remember number of actually selected term (sql generation may append some) if cachekey is None: self.no_cache += 1 @@ -336,10 +331,9 @@ self.info("request failed '%s' ... retry with a new cursor", sql) session.pool.reconnect(self) cursor = self.doexec(session, sql, args) - res = self.process_result(cursor) - if server.DEBUG & (server.DBG_SQL | server.DBG_RQL): - print '------>', res - return res + results = self.process_result(cursor) + assert dbg_results(results) + return results def flying_insert(self, table, session, union, args=None, varmap=None): """similar as .syntax_tree_search, but inserts data in the @@ -347,23 +341,18 @@ source whose the given cursor come from). If not possible, inserts all data by calling .executemany(). """ - if self.uri == 'system': - if server.DEBUG & server.DBG_RQL: - print 'FLYING RQL FOR SOURCE %s: %s', self.uri, union.as_string() - if varmap: - print 'USING VARMAP', varmap - if server.DEBUG & server.DBG_MORE: - print 'SOLUTIONS', ','.join(str(s.solutions) for s in union.children) - # generate sql queries if we are able to do so - sql, query_args = self._rql_sqlgen.generate(union, args, varmap) - query = 'INSERT INTO %s %s' % (table, sql.encode(self.encoding)) - self.doexec(session, query, self.merge_args(args, query_args)) - else: - super(NativeSQLSource, self).flying_insert(table, session, union, - args, varmap) + assert dbg_st_search( + uri, union, varmap, args, + prefix='ON THE FLY temp data insertion into %s from' % table) + # generate sql queries if we are able to do so + sql, query_args = self._rql_sqlgen.generate(union, args, varmap) + query = 'INSERT INTO %s %s' % (table, sql.encode(self.encoding)) + self.doexec(session, query, self.merge_args(args, query_args)) def _manual_insert(self, results, table, session): """insert given result into a temporary table on the system source""" + if server.DEBUG & server.DBG_RQL: + print ' manual insertion of', res, 'into', table if not results: return query_args = ['%%(%s)s' % i for i in xrange(len(results[0]))] @@ -436,7 +425,10 @@ """ cursor = session.pool[self.uri] if server.DEBUG & server.DBG_SQL: - print 'exec', query, args + cnx = session.pool.connection(self.uri) + # getattr to get the actual connection if cnx is a ConnectionWrapper + # instance + print 'exec', query, args, getattr(cnx, '_cnx', cnx) try: # str(query) to avoid error if it's an unicode string cursor.execute(str(query), args) diff -r c0b5b189190e -r d6012db7b93e server/sources/pyrorql.py --- a/server/sources/pyrorql.py Sat Aug 01 01:24:42 2009 +0200 +++ b/server/sources/pyrorql.py Sat Aug 01 16:13:40 2009 +0200 @@ -23,7 +23,8 @@ from cubicweb import dbapi, server from cubicweb import BadConnectionId, UnknownEid, ConnectionError from cubicweb.cwconfig import register_persistent_options -from cubicweb.server.sources import AbstractSource, ConnectionWrapper, TimedCache +from cubicweb.server.sources import (AbstractSource, ConnectionWrapper, + TimedCache, dbg_st_search, dbg_results) class ReplaceByInOperator(Exception): def __init__(self, eids): @@ -253,13 +254,14 @@ def syntax_tree_search(self, session, union, args=None, cachekey=None, varmap=None): - #assert not varmap, (varmap, union) + assert dbg_st_search(self.uri, union, varmap, args, cachekey) rqlkey = union.as_string(kwargs=args) try: results = self._query_cache[rqlkey] except KeyError: results = self._syntax_tree_search(session, union, args) self._query_cache[rqlkey] = results + assert dbg_results(results) return results def _syntax_tree_search(self, session, union, args): @@ -270,12 +272,6 @@ """ if not args is None: args = args.copy() - if server.DEBUG & server.DBG_RQL: - print 'RQL FOR PYRO SOURCE %s: %s', self.uri, union.as_string() - if args: - print 'ARGS', args - if server.DEBUG & server.DBG_MORE: - print 'SOLUTIONS', ','.join(str(s.solutions) for s in union.children) # get cached cursor anyway cu = session.pool[self.uri] if cu is None: @@ -287,10 +283,10 @@ rql, cachekey = RQL2RQL(self).generate(session, union, args) except UnknownEid, ex: if server.DEBUG: - print 'unknown eid', ex, 'no results' + print ' unknown eid', ex, 'no results' return [] if server.DEBUG & server.DBG_RQL: - print 'TRANSLATED RQL', rql + print ' translated rql', rql try: rset = cu.execute(rql, args, cachekey) except Exception, ex: @@ -326,11 +322,6 @@ results = rows else: results = [] - if server.DEBUG & server.DBG_RQL: - if len(results)>10: - print '-->', results[:10], '...', len(results) - else: - print '-->', results return results def _entity_relations_and_kwargs(self, session, entity):