# HG changeset patch # User Sylvain Thénault # Date 1461926348 -7200 # Node ID f32134dd0067a2dab87361f2a6b490bbd2d54a5c # Parent 25ec9be5f3053f93f3307fffd46b83824d65d639 [repository] drop remanescence of old multi-sources code we don't use anymore this inputmap/varmap thing. diff -r 25ec9be5f305 -r f32134dd0067 cubicweb/devtools/repotest.py --- a/cubicweb/devtools/repotest.py Mon Mar 21 17:10:08 2016 +0100 +++ b/cubicweb/devtools/repotest.py Fri Apr 29 12:39:08 2016 +0200 @@ -245,7 +245,7 @@ plan = self._prepare_plan(cnx, rql, kwargs, simplify=False) plan.preprocess(plan.rqlst) rqlst = plan.rqlst.children[0] - rqlst.solutions = remove_unused_solutions(rqlst, rqlst.solutions, {}, self.repo.schema)[0] + rqlst.solutions = remove_unused_solutions(rqlst, rqlst.solutions, self.repo.schema)[0] return rqlst def user_groups_session(self, *groups): diff -r 25ec9be5f305 -r f32134dd0067 cubicweb/server/querier.py --- a/cubicweb/server/querier.py Mon Mar 21 17:10:08 2016 +0100 +++ b/cubicweb/server/querier.py Fri Apr 29 12:39:08 2016 +0200 @@ -51,15 +51,6 @@ """build an empty result set object""" return ResultSet([], rql, args, rqlst=rqlst) -def update_varmap(varmap, selected, table): - """return a sql schema to store RQL query result""" - for i, term in enumerate(selected): - key = term.as_string() - value = '%s.C%s' % (table, i) - if varmap.get(key, value) != value: - raise Exception('variable name conflict on %s: got %s / %s' - % (key, value, varmap)) - varmap[key] = value # permission utilities ######################################################## diff -r 25ec9be5f305 -r f32134dd0067 cubicweb/server/sources/__init__.py --- a/cubicweb/server/sources/__init__.py Mon Mar 21 17:10:08 2016 +0100 +++ b/cubicweb/server/sources/__init__.py Fri Apr 29 12:39:08 2016 +0200 @@ -37,13 +37,11 @@ from cubicweb.server.edition import EditedEntity -def dbg_st_search(uri, union, varmap, args, cachekey=None, prefix='rql for'): +def dbg_st_search(uri, union, args, cachekey=None, prefix='rql for'): if server.DEBUG & server.DBG_RQL: global t print(' %s %s source: %s' % (prefix, uri, repr(union.as_string()))) t = time() - if varmap: - print(' using varmap', varmap) if server.DEBUG & server.DBG_MORE: print(' args', repr(args)) print(' cache key', cachekey) @@ -361,7 +359,7 @@ # RQL query api ############################################################ def syntax_tree_search(self, cnx, union, - args=None, cachekey=None, varmap=None, debug=0): + args=None, cachekey=None, debug=0): """return result from this source for a rql query (actually from a rql syntax tree and a solution dictionary mapping each used variable to a possible type). If cachekey is given, the query necessary to fetch the diff -r 25ec9be5f305 -r f32134dd0067 cubicweb/server/sources/native.py --- a/cubicweb/server/sources/native.py Mon Mar 21 17:10:08 2016 +0100 +++ b/cubicweb/server/sources/native.py Fri Apr 29 12:39:08 2016 +0200 @@ -532,20 +532,19 @@ continue raise AuthenticationError() - def syntax_tree_search(self, cnx, union, args=None, cachekey=None, - varmap=None): + def syntax_tree_search(self, cnx, union, args=None, cachekey=None): """return result from this source for a rql query (actually from a rql syntax tree and a solution dictionary mapping each used variable to a possible type). If cachekey is given, the query necessary to fetch the results (but not the results themselves) may be cached using this key. """ - assert dbg_st_search(self.uri, union, varmap, args, cachekey) + assert dbg_st_search(self.uri, union, args, cachekey) # remember number of actually selected term (sql generation may append some) if cachekey is None: self.no_cache += 1 # generate sql query if we are able to do so (not supported types...) - sql, qargs, cbs = self._rql_sqlgen.generate(union, args, varmap) + sql, qargs, cbs = self._rql_sqlgen.generate(union, args) else: # sql may be cached try: @@ -553,7 +552,7 @@ self.cache_hit += 1 except KeyError: self.cache_miss += 1 - sql, qargs, cbs = self._rql_sqlgen.generate(union, args, varmap) + sql, qargs, cbs = self._rql_sqlgen.generate(union, args) self._cache[cachekey] = sql, qargs, cbs args = self.merge_args(args, qargs) assert isinstance(sql, string_types), repr(sql) diff -r 25ec9be5f305 -r f32134dd0067 cubicweb/server/sources/rql2sql.py --- a/cubicweb/server/sources/rql2sql.py Mon Mar 21 17:10:08 2016 +0100 +++ b/cubicweb/server/sources/rql2sql.py Fri Apr 29 12:39:08 2016 +0200 @@ -167,7 +167,7 @@ newsolutions.append(asol) return newsolutions -def remove_unused_solutions(rqlst, solutions, varmap, schema): +def remove_unused_solutions(rqlst, solutions, schema): """cleanup solutions: remove solutions where invariant variables are taking different types """ @@ -177,7 +177,7 @@ invariants = {} for vname, var in rqlst.defined_vars.items(): vtype = newsols[0][vname] - if var._q_invariant or vname in varmap: + if var._q_invariant: # remove invariant variable from solutions to remove duplicates # later, then reinserting a type for the variable even later for sol in newsols: @@ -376,7 +376,6 @@ self.aliases = {} self.restrictions = [] self._restr_stack = [] - self.ignore_varmap = False self._needs_source_cb = {} def merge_source_cbs(self, needs_source_cb): @@ -715,23 +714,18 @@ attrmap = {} self.attr_map = attrmap - def generate(self, union, args=None, varmap=None): + def generate(self, union, args=None): """return SQL queries and a variable dictionary from a RQL syntax tree :partrqls: a list of couple (rqlst, solutions) :args: optional dictionary with values of substitutions used in the query - :varmap: optional dictionary mapping variable name to a special table - name, in case the query as to fetch data from temporary tables return an sql string and a dictionary with substitutions values """ if args is None: args = {} - if varmap is None: - varmap = {} self._lock.acquire() self._args = args - self._varmap = varmap self._query_attrs = {} self._state = None # self._not_scope_offset = 0 @@ -803,7 +797,7 @@ if len(sols) > 1: # remove invariant from solutions sols, existssols, unstable = remove_unused_solutions( - select, sols, self._varmap, self.schema) + select, sols, self.schema) if len(sols) > 1: # if there is still more than one solution, a UNION will be # generated and so sort terms have to be selected @@ -1048,15 +1042,8 @@ sql = self._visit_attribute_relation(relation) elif (rtype == 'is' and isinstance(rhs.children[0], Constant) and rhs.children[0].eval(self._args) is None): - # special case "C is NULL" - if lhs.name in self._varmap: - lhssql = self._varmap[lhs.name] - else: - lhssql = lhs.accept(self) + lhssql = lhs.accept(self) return '%s%s' % (lhssql, rhs.accept(self)) - elif '%s.%s' % (lhs, relation.r_type) in self._varmap: - # relation has already been processed by a previous step - return '' elif relation.optional: # OPTIONAL relation, generate a left|right outer join if rtype == 'identity' or rschema.inlined: @@ -1088,8 +1075,7 @@ lhssql, lhssql, (rhsvar or rhsconst).accept(self)) elif rhsconst is not None: sql = '%s=%s' % (lhssql, rhsconst.accept(self)) - elif isinstance(rhsvar, Variable) and rhsvar._q_invariant and \ - not rhsvar.name in self._varmap: + elif isinstance(rhsvar, Variable) and rhsvar._q_invariant: # if the rhs variable is only linked to this relation, this mean we # only want the relation to exists, eg NOT NULL in case of inlined # relation @@ -1107,10 +1093,6 @@ termsql = termconst and termconst.accept(self) or termvar.accept(self) yield '%s.%s=%s' % (rid, relfield, termsql) elif termvar._q_invariant: - # if the variable is mapped, generate restriction anyway - if termvar.name in self._varmap: - termsql = termvar.accept(self) - yield '%s.%s=%s' % (rid, relfield, termsql) extrajoin = self._extra_join_sql(relation, '%s.%s' % (rid, relfield), termvar) if extrajoin is not None: yield extrajoin @@ -1236,15 +1218,12 @@ attr = 'eid' if relation.r_type == 'identity' else relation.r_type lhsalias = self._var_table(lhsvar) rhsalias = rhsvar and self._var_table(rhsvar) - try: - lhssql = self._varmap['%s.%s' % (lhsvar.name, attr)] - except KeyError: - if lhsalias is None: - lhssql = lhsconst.accept(self) - elif attr == 'eid': - lhssql = lhsvar.accept(self) - else: - lhssql = '%s.%s%s' % (lhsalias, SQL_PREFIX, attr) + if lhsalias is None: + lhssql = lhsconst.accept(self) + elif attr == 'eid': + lhssql = lhsvar.accept(self) + else: + lhssql = '%s.%s%s' % (lhsalias, SQL_PREFIX, attr) condition = '%s=%s' % (lhssql, (rhsconst or rhsvar).accept(self)) # this is not a typo, rhs optional variable means lhs outer join and vice-versa if relation.optional == 'left': @@ -1285,9 +1264,6 @@ ored = relation.ored() for vref in rhs_vars: var = vref.variable - if var.name in self._varmap: - # ensure table is added - self._var_info(var) if isinstance(var, ColumnAlias): # force sql generation whatever the computed principal principal = 1 @@ -1308,11 +1284,7 @@ _rel = relation lhssql = self._inlined_var_sql(_rel.children[0].variable, _rel.r_type) - try: - self._state.ignore_varmap = True - sql = lhssql + relation.children[1].accept(self) - finally: - self._state.ignore_varmap = False + sql = lhssql + relation.children[1].accept(self) if relation.optional == 'right': leftalias = self._var_table(principal.children[0].variable) rightalias = self._var_table(relation.children[0].variable) @@ -1331,22 +1303,19 @@ assert rel.r_type == 'eid' lhssql = lhs.accept(self) else: - try: - lhssql = self._varmap['%s.%s' % (lhs.name, rel.r_type)] - except KeyError: - mapkey = '%s.%s' % (self._state.solution[lhs.name], rel.r_type) - if mapkey in self.attr_map: - cb, sourcecb = self.attr_map[mapkey] - if sourcecb: - # callback is a source callback, we can't use this - # attribute in restriction - raise QueryError("can't use %s (%s) in restriction" - % (mapkey, rel.as_string())) - lhssql = cb(self, lhs.variable, rel) - elif rel.r_type == 'eid': - lhssql = lhs.variable._q_sql - else: - lhssql = '%s.%s%s' % (table, SQL_PREFIX, rel.r_type) + mapkey = '%s.%s' % (self._state.solution[lhs.name], rel.r_type) + if mapkey in self.attr_map: + cb, sourcecb = self.attr_map[mapkey] + if sourcecb: + # callback is a source callback, we can't use this + # attribute in restriction + raise QueryError("can't use %s (%s) in restriction" + % (mapkey, rel.as_string())) + lhssql = cb(self, lhs.variable, rel) + elif rel.r_type == 'eid': + lhssql = lhs.variable._q_sql + else: + lhssql = '%s.%s%s' % (table, SQL_PREFIX, rel.r_type) try: if rel._q_needcast == 'TODAY': sql = 'DATE(%s)%s' % (lhssql, rhssql) @@ -1375,7 +1344,7 @@ if lhsvar.stinfo['typerel'] is None: # the variable is using the fti table, no join needed jointo = None - elif not lhsvar.name in self._varmap: + else: # join on entities instead of etype's table to get result for # external entities on multisources configurations ealias = lhsvar._q_sqltable = '_' + lhsvar.name @@ -1510,13 +1479,9 @@ rel._q_needcast = value return self.keyword_map[value]() if constant.type == 'Substitute': - try: - # we may found constant from simplified var in varmap - return self._mapped_term(constant, '%%(%s)s' % value)[0] - except KeyError: - _id = value - if PY2 and isinstance(_id, unicode): - _id = _id.encode() + _id = value + if PY2 and isinstance(_id, unicode): + _id = _id.encode() else: _id = str(id(constant)).replace('-', '', 1) self._query_attrs[_id] = value @@ -1529,13 +1494,6 @@ def visit_columnalias(self, colalias): """get the sql name for a subquery column alias""" - if colalias.name in self._varmap: - sql = self._varmap[colalias.name] - table = sql.split('.', 1)[0] - colalias._q_sqltable = table - colalias._q_sql = sql - self._state.add_table(table) - return sql return colalias._q_sql def visit_variable(self, variable): @@ -1547,9 +1505,7 @@ return variable._q_sql self._state.done.add(variable.name) vtablename = None - if not self._state.ignore_varmap and variable.name in self._varmap: - sql, vtablename = self._var_info(variable) - elif variable.stinfo['attrvar']: + if variable.stinfo['attrvar']: # attribute variable (systematically used in rhs of final # relation(s)), get table name and sql from any rhs relation sql = self._linked_var_sql(variable) @@ -1610,66 +1566,30 @@ pass return None - def _temp_table_scope(self, select, table): - scope = 9999 - for var, sql in self._varmap.items(): - # skip "attribute variable" in varmap (such 'T.login') - if not '.' in var and table == sql.split('.', 1)[0]: - try: - scope = min(scope, self._state.scopes[select.defined_vars[var].scope]) - except KeyError: - scope = 0 # XXX - if scope == 0: - break - return scope - - def _mapped_term(self, term, key): - """return sql and table alias to the `term`, mapped as `key` or raise - KeyError when the key is not found in the varmap - """ - sql = self._varmap[key] - tablealias = sql.split('.', 1)[0] - scope = self._temp_table_scope(term.stmt, tablealias) - self._state.add_table(tablealias, scope=scope) - return sql, tablealias - def _var_info(self, var): - try: - return self._mapped_term(var, var.name) - except KeyError: - scope = self._state.scopes[var.scope] - etype = self._state.solution[var.name] - # XXX this check should be moved in rql.stcheck - if self.schema.eschema(etype).final: - raise BadRQLQuery(var.stmt.root) - tablealias = '_' + var.name - sql = '%s.%seid' % (tablealias, SQL_PREFIX) - self._state.add_table('%s%s AS %s' % (SQL_PREFIX, etype, tablealias), - tablealias, scope=scope) + scope = self._state.scopes[var.scope] + etype = self._state.solution[var.name] + # XXX this check should be moved in rql.stcheck + if self.schema.eschema(etype).final: + raise BadRQLQuery(var.stmt.root) + tablealias = '_' + var.name + sql = '%s.%seid' % (tablealias, SQL_PREFIX) + self._state.add_table('%s%s AS %s' % (SQL_PREFIX, etype, tablealias), + tablealias, scope=scope) return sql, tablealias def _inlined_var_sql(self, var, rtype): - try: - sql = self._varmap['%s.%s' % (var.name, rtype)] - scope = self._state.scopes[var.scope] - self._state.add_table(sql.split('.', 1)[0], scope=scope) - except KeyError: - # rtype may be an attribute relation when called from - # _visit_var_attr_relation. take care about 'eid' rtype, since in - # some case we may use the `entities` table, so in that case we've - # to properly use variable'sql - if rtype == 'eid': - sql = var.accept(self) - else: - sql = '%s.%s%s' % (self._var_table(var), SQL_PREFIX, rtype) + # rtype may be an attribute relation when called from + # _visit_var_attr_relation. take care about 'eid' rtype, since in + # some case we may use the `entities` table, so in that case we've + # to properly use variable'sql + if rtype == 'eid': + sql = var.accept(self) + else: + sql = '%s.%s%s' % (self._var_table(var), SQL_PREFIX, rtype) return sql def _linked_var_sql(self, variable): - if not self._state.ignore_varmap: - try: - return self._varmap[variable.name] - except KeyError: - pass rel = (variable.stinfo.get('principal') or next(iter(variable.stinfo['rhsrelations']))) linkedvar = rel.children[0].variable @@ -1678,23 +1598,19 @@ if isinstance(linkedvar, ColumnAlias): raise BadRQLQuery('variable %s should be selected by the subquery' % variable.name) - try: - sql = self._varmap['%s.%s' % (linkedvar.name, rel.r_type)] - except KeyError: - mapkey = '%s.%s' % (self._state.solution[linkedvar.name], rel.r_type) - if mapkey in self.attr_map: - cb, sourcecb = self.attr_map[mapkey] - if not sourcecb: - return cb(self, linkedvar, rel) - # attribute mapped at the source level (bfss for instance) - stmt = rel.stmt - for selectidx, vref in iter_mapped_var_sels(stmt, variable): - stack = [cb] - update_source_cb_stack(self._state, stmt, vref, stack) - self._state._needs_source_cb[selectidx] = stack - linkedvar.accept(self) - sql = '%s.%s%s' % (linkedvar._q_sqltable, SQL_PREFIX, rel.r_type) - return sql + mapkey = '%s.%s' % (self._state.solution[linkedvar.name], rel.r_type) + if mapkey in self.attr_map: + cb, sourcecb = self.attr_map[mapkey] + if not sourcecb: + return cb(self, linkedvar, rel) + # attribute mapped at the source level (bfss for instance) + stmt = rel.stmt + for selectidx, vref in iter_mapped_var_sels(stmt, variable): + stack = [cb] + update_source_cb_stack(self._state, stmt, vref, stack) + self._state._needs_source_cb[selectidx] = stack + linkedvar.accept(self) + return '%s.%s%s' % (linkedvar._q_sqltable, SQL_PREFIX, rel.r_type) # tables handling ######################################################### diff -r 25ec9be5f305 -r f32134dd0067 cubicweb/server/ssplanner.py --- a/cubicweb/server/ssplanner.py Mon Mar 21 17:10:08 2016 +0100 +++ b/cubicweb/server/ssplanner.py Fri Apr 29 12:39:08 2016 +0200 @@ -304,15 +304,6 @@ # execution steps and helper functions ######################################## -def varmap_test_repr(varmap, tablesinorder): - if varmap is None: - return varmap - maprepr = {} - for var, sql in varmap.items(): - table, col = sql.split('.') - maprepr[var] = '%s.%s' % (tablesinorder[table], col) - return maprepr - class Step(object): """base abstract class for execution step""" def __init__(self, plan): @@ -345,10 +336,9 @@ """step consisting in fetching data from sources and directly returning results """ - def __init__(self, plan, union, inputmap=None): + def __init__(self, plan, union): Step.__init__(self, plan) self.union = union - self.inputmap = inputmap def execute(self): """call .syntax_tree_search with the given syntax tree on each @@ -357,11 +347,8 @@ self.execute_children() cnx = self.plan.cnx args = self.plan.args - inputmap = self.inputmap union = self.union - # do we have to use a inputmap from a previous step ? If so disable - # cachekey - if inputmap or self.plan.cache_key is None: + if self.plan.cache_key is None: cachekey = None # union may have been splited into subqueries, in which case we can't # use plan.cache_key, rebuild a cache key @@ -373,20 +360,15 @@ cachekey = union.as_string() # get results for query source = cnx.repo.system_source - result = source.syntax_tree_search(cnx, union, args, cachekey, inputmap) + result = source.syntax_tree_search(cnx, union, args, cachekey) #print 'ONEFETCH RESULT %s' % (result) return result def mytest_repr(self): """return a representation of this step suitable for test""" - try: - inputmap = varmap_test_repr(self.inputmap, self.plan.tablesinorder) - except AttributeError: - inputmap = self.inputmap return (self.__class__.__name__, sorted((r.as_string(kwargs=self.plan.args), r.solutions) - for r in self.union.children), - inputmap) + for r in self.union.children)) # UPDATE/INSERT/DELETE steps ################################################## diff -r 25ec9be5f305 -r f32134dd0067 cubicweb/server/test/unittest_rql2sql.py --- a/cubicweb/server/test/unittest_rql2sql.py Mon Mar 21 17:10:08 2016 +0100 +++ b/cubicweb/server/test/unittest_rql2sql.py Fri Apr 29 12:39:08 2016 +0200 @@ -1235,13 +1235,12 @@ def _norm_sql(self, sql): return sql.strip() - def _check(self, rql, sql, varmap=None, args=None): + def _check(self, rql, sql, args=None): if args is None: args = {'text': 'hip hop momo', 'eid': 12345} try: union = self._prepare(rql) - r, nargs, cbs = self.o.generate(union, args, - varmap=varmap) + r, nargs, cbs = self.o.generate(union, args) args.update(nargs) self.assertMultiLineEqual(strip(r % args), self._norm_sql(sql)) except Exception as ex: @@ -1303,26 +1302,6 @@ FROM in_basket_relation AS rel_in_basket0 WHERE rel_in_basket0.eid_to=12''') - def test_varmap1(self): - self._check('Any X,L WHERE X is CWUser, X in_group G, X login L, G name "users"', - '''SELECT T00.x, T00.l -FROM T00, cw_CWGroup AS _G, in_group_relation AS rel_in_group0 -WHERE rel_in_group0.eid_from=T00.x AND rel_in_group0.eid_to=_G.cw_eid AND _G.cw_name=users''', - varmap={'X': 'T00.x', 'X.login': 'T00.l'}) - - def test_varmap2(self): - self._check('Any X,L,GN WHERE X is CWUser, X in_group G, X login L, G name GN', - '''SELECT T00.x, T00.l, _G.cw_name -FROM T00, cw_CWGroup AS _G, in_group_relation AS rel_in_group0 -WHERE rel_in_group0.eid_from=T00.x AND rel_in_group0.eid_to=_G.cw_eid''', - varmap={'X': 'T00.x', 'X.login': 'T00.l'}) - - def test_varmap3(self): - self._check('Any %(x)s,D WHERE F data D, F is File', - 'SELECT 728, _TDF0.C0\nFROM _TDF0', - args={'x': 728}, - varmap={'F.data': '_TDF0.C0', 'D': '_TDF0.C0'}) - def test_is_null_transform(self): union = self._prepare('Any X WHERE X login %(login)s') r, args, cbs = self.o.generate(union, {'login': None}) @@ -2231,7 +2210,7 @@ rqlst.defined_vars['A'] = mock_object(scope=rqlst, stinfo={}, _q_invariant=True) rqlst.defined_vars['B'] = mock_object(scope=rqlst, stinfo={}, _q_invariant=False) self.assertEqual(remove_unused_solutions(rqlst, [{'A': 'RugbyGroup', 'B': 'RugbyTeam'}, - {'A': 'FootGroup', 'B': 'FootTeam'}], {}, None), + {'A': 'FootGroup', 'B': 'FootTeam'}], None), ([{'A': 'RugbyGroup', 'B': 'RugbyTeam'}, {'A': 'FootGroup', 'B': 'FootTeam'}], {}, set('B')) @@ -2242,7 +2221,7 @@ rqlst.defined_vars['A'] = mock_object(scope=rqlst, stinfo={}, _q_invariant=True) rqlst.defined_vars['B'] = mock_object(scope=rqlst, stinfo={}, _q_invariant=False) self.assertEqual(remove_unused_solutions(rqlst, [{'A': 'RugbyGroup', 'B': 'RugbyTeam'}, - {'A': 'FootGroup', 'B': 'RugbyTeam'}], {}, None), + {'A': 'FootGroup', 'B': 'RugbyTeam'}], None), ([{'A': 'RugbyGroup', 'B': 'RugbyTeam'}], {}, set()) ) diff -r 25ec9be5f305 -r f32134dd0067 cubicweb/server/test/unittest_ssplanner.py --- a/cubicweb/server/test/unittest_ssplanner.py Mon Mar 21 17:10:08 2016 +0100 +++ b/cubicweb/server/test/unittest_ssplanner.py Fri Apr 29 12:39:08 2016 +0200 @@ -51,7 +51,7 @@ [{'X': 'Basket', 'XN': 'String'}, {'X': 'State', 'XN': 'String'}, {'X': 'Folder', 'XN': 'String'}])], - None, [])]) + [])]) def test_groupeded_ambigous_sol(self): self._test('Any XN,COUNT(X) GROUPBY XN WHERE X name XN, X is IN (Basket, State, Folder)', @@ -59,7 +59,7 @@ [{'X': 'Basket', 'XN': 'String'}, {'X': 'State', 'XN': 'String'}, {'X': 'Folder', 'XN': 'String'}])], - None, [])]) + [])]) if __name__ == '__main__': from logilab.common.testlib import unittest_main