# HG changeset patch # User Sylvain Thénault # Date 1373878774 -7200 # Node ID c05652b108cecc541b66fd979e90640eca56ae2e # Parent e47e192ea0d98456ca68021e7522c7406aab01ee [rql rewrite] move some code from querier to rqlrewrite where it makes more sense. Also, make some minor cleanup/refactoring on the way and try to enhance docstrings. Relates to #3013535 diff -r e47e192ea0d9 -r c05652b108ce rqlrewrite.py --- a/rqlrewrite.py Fri Jul 12 10:39:01 2013 +0200 +++ b/rqlrewrite.py Mon Jul 15 10:59:34 2013 +0200 @@ -1,4 +1,4 @@ -# copyright 2003-2012 LOGILAB S.A. (Paris, FRANCE), all rights reserved. +# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved. # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr # # This file is part of CubicWeb. @@ -33,6 +33,13 @@ from cubicweb import Unauthorized +def cleanup_solutions(rqlst, solutions): + for sol in solutions: + for vname in list(sol): + if not (vname in rqlst.defined_vars or vname in rqlst.aliases): + del sol[vname] + + def add_types_restriction(schema, rqlst, newroot=None, solutions=None): if newroot is None: assert solutions is None @@ -132,10 +139,35 @@ return newsolutions +def _add_noinvariant(noinvariant, restricted, select, nbtrees): + # a variable can actually be invariant if it has not been restricted for + # security reason or if security assertion hasn't modified the possible + # solutions for the query + for vname in restricted: + try: + var = select.defined_vars[vname] + except KeyError: + # this is an alias + continue + if nbtrees != 1 or len(var.stinfo['possibletypes']) != 1: + noinvariant.add(var) + + +def _expand_selection(terms, selected, aliases, select, newselect): + for term in terms: + for vref in term.iget_nodes(n.VariableRef): + if not vref.name in selected: + select.append_selected(vref) + colalias = newselect.get_variable(vref.name, len(aliases)) + aliases.append(n.VariableRef(colalias)) + selected.add(vref.name) + + def iter_relations(stinfo): # this is a function so that test may return relation in a predictable order return stinfo['relations'] - stinfo['rhsrelations'] + class Unsupported(Exception): """raised when an rql expression can't be inserted in some rql query because it create an unresolvable query (eg no solutions found) @@ -164,6 +196,110 @@ if len(self.select.solutions) < len(self.solutions): raise Unsupported() + def insert_local_checks(self, select, kwargs, + localchecks, restricted, noinvariant): + """ + select: the rql syntax tree Select node + kwargs: query arguments + + localchecks: {(('Var name', (rqlexpr1, rqlexpr2)), + ('Var name1', (rqlexpr1, rqlexpr23))): [solution]} + + (see querier._check_permissions docstring for more information) + + restricted: set of variable names to which an rql expression has to be + applied + + noinvariant: set of variable names that can't be considered has + invariant due to security reason (will be filed by this method) + """ + nbtrees = len(localchecks) + myunion = union = select.parent + # transform in subquery when len(localchecks)>1 and groups + if nbtrees > 1 and (select.orderby or select.groupby or + select.having or select.has_aggregat or + select.distinct or + select.limit or select.offset): + newselect = stmts.Select() + # only select variables in subqueries + origselection = select.selection + select.select_only_variables() + select.has_aggregat = False + # create subquery first so correct node are used on copy + # (eg ColumnAlias instead of Variable) + aliases = [n.VariableRef(newselect.get_variable(vref.name, i)) + for i, vref in enumerate(select.selection)] + selected = set(vref.name for vref in aliases) + # now copy original selection and groups + for term in origselection: + newselect.append_selected(term.copy(newselect)) + if select.orderby: + sortterms = [] + for sortterm in select.orderby: + sortterms.append(sortterm.copy(newselect)) + for fnode in sortterm.get_nodes(n.Function): + if fnode.name == 'FTIRANK': + # we've to fetch the has_text relation as well + var = fnode.children[0].variable + rel = iter(var.stinfo['ftirels']).next() + assert not rel.ored(), 'unsupported' + newselect.add_restriction(rel.copy(newselect)) + # remove relation from the orig select and + # cleanup variable stinfo + rel.parent.remove(rel) + var.stinfo['ftirels'].remove(rel) + var.stinfo['relations'].remove(rel) + # XXX not properly re-annotated after security insertion? + newvar = newselect.get_variable(var.name) + newvar.stinfo.setdefault('ftirels', set()).add(rel) + newvar.stinfo.setdefault('relations', set()).add(rel) + newselect.set_orderby(sortterms) + _expand_selection(select.orderby, selected, aliases, select, newselect) + select.orderby = () # XXX dereference? + if select.groupby: + newselect.set_groupby([g.copy(newselect) for g in select.groupby]) + _expand_selection(select.groupby, selected, aliases, select, newselect) + select.groupby = () # XXX dereference? + if select.having: + newselect.set_having([g.copy(newselect) for g in select.having]) + _expand_selection(select.having, selected, aliases, select, newselect) + select.having = () # XXX dereference? + if select.limit: + newselect.limit = select.limit + select.limit = None + if select.offset: + newselect.offset = select.offset + select.offset = 0 + myunion = stmts.Union() + newselect.set_with([n.SubQuery(aliases, myunion)], check=False) + newselect.distinct = select.distinct + solutions = [sol.copy() for sol in select.solutions] + cleanup_solutions(newselect, solutions) + newselect.set_possible_types(solutions) + # if some solutions doesn't need rewriting, insert original + # select as first union subquery + if () in localchecks: + myunion.append(select) + # we're done, replace original select by the new select with + # subqueries (more added in the loop below) + union.replace(select, newselect) + elif not () in localchecks: + union.remove(select) + for lcheckdef, lchecksolutions in localchecks.iteritems(): + if not lcheckdef: + continue + myrqlst = select.copy(solutions=lchecksolutions) + myunion.append(myrqlst) + # in-place rewrite + annotation / simplification + lcheckdef = [({var: 'X'}, rqlexprs) for var, rqlexprs in lcheckdef] + self.rewrite(myrqlst, lcheckdef, lchecksolutions, kwargs) + _add_noinvariant(noinvariant, restricted, myrqlst, nbtrees) + if () in localchecks: + select.set_possible_types(localchecks[()]) + add_types_restriction(self.schema, select) + _add_noinvariant(noinvariant, restricted, select, nbtrees) + self.annotate(union) + def rewrite(self, select, snippets, solutions, kwargs, existingvars=None): """ snippets: (varmap, list of rql expression) diff -r e47e192ea0d9 -r c05652b108ce server/msplanner.py --- a/server/msplanner.py Fri Jul 12 10:39:01 2013 +0200 +++ b/server/msplanner.py Mon Jul 15 10:59:34 2013 +0200 @@ -100,8 +100,7 @@ from cubicweb import server from cubicweb.utils import make_uid -from cubicweb.rqlrewrite import add_types_restriction -from cubicweb.server.utils import cleanup_solutions +from cubicweb.rqlrewrite import add_types_restriction, cleanup_solutions from cubicweb.server.ssplanner import SSPlanner, OneFetchStep from cubicweb.server.mssteps import * diff -r e47e192ea0d9 -r c05652b108ce server/querier.py --- a/server/querier.py Fri Jul 12 10:39:01 2013 +0200 +++ b/server/querier.py Mon Jul 15 10:59:34 2013 +0200 @@ -1,4 +1,4 @@ -# copyright 2003-2012 LOGILAB S.A. (Paris, FRANCE), all rights reserved. +# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved. # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr # # This file is part of CubicWeb. @@ -24,18 +24,15 @@ from logilab.common.compat import any from rql import RQLSyntaxError, CoercionError -from rql.stmts import Union, Select -from rql.nodes import ETYPE_PYOBJ_MAP, etype_from_pyobj -from rql.nodes import (Relation, VariableRef, Constant, SubQuery, Function, - Exists, Not) +from rql.stmts import Union +from rql.nodes import ETYPE_PYOBJ_MAP, etype_from_pyobj, Relation, Exists, Not from yams import BASE_TYPES -from cubicweb import ValidationError, Unauthorized, QueryError, UnknownEid +from cubicweb import ValidationError, Unauthorized, UnknownEid from cubicweb import Binary, server from cubicweb.rset import ResultSet from cubicweb.utils import QueryCache, RepeatList -from cubicweb.server.utils import cleanup_solutions from cubicweb.server.rqlannotation import SQLGenAnnotator, set_qdata from cubicweb.server.ssplanner import READ_ONLY_RTYPES, add_types_restriction from cubicweb.server.edition import EditedEntity @@ -77,12 +74,13 @@ return session.describe(term.eval(args))[0] def check_read_access(session, rqlst, solution, args): - """check that the given user has credentials to access data read the - query + """Check that the given user has credentials to access data read by the + query and return a dict defining necessary "local checks" (i.e. rql + expression in read permission defined in the schema) where no group grants + him the permission. - return a dict defining necessary local checks (due to use of rql expression - in the schema), keys are variable names and values associated rql expression - for the associated variable with the given solution + Returned dictionary's keys are variable names and values the rql expressions + for this variable (with the given solution). """ # use `term_etype` since we've to deal with rewritten constants here, # when used as an external source by another repository. @@ -130,35 +128,6 @@ localchecks[varname] = erqlexprs return localchecks -def add_noinvariant(noinvariant, restricted, select, nbtrees): - # a variable can actually be invariant if it has not been restricted for - # security reason or if security assertion hasn't modified the possible - # solutions for the query - if nbtrees != 1: - for vname in restricted: - try: - noinvariant.add(select.defined_vars[vname]) - except KeyError: - # this is an alias - continue - else: - for vname in restricted: - try: - var = select.defined_vars[vname] - except KeyError: - # this is an alias - continue - if len(var.stinfo['possibletypes']) != 1: - noinvariant.add(var) - -def _expand_selection(terms, selected, aliases, select, newselect): - for term in terms: - for vref in term.iget_nodes(VariableRef): - if not vref.name in selected: - select.append_selected(vref) - colalias = newselect.get_variable(vref.name, len(aliases)) - aliases.append(VariableRef(colalias)) - selected.add(vref.name) # Plans ####################################################################### @@ -258,9 +227,8 @@ self.args = args cached = True else: - noinvariant = set() with self.session.security_enabled(read=False): - self._insert_security(union, noinvariant) + noinvariant = self._insert_security(union) if key is not None: self.session.transaction_data[key] = (union, self.args) else: @@ -272,121 +240,39 @@ if union.has_text_query: self.cache_key = None - def _insert_security(self, union, noinvariant): + def _insert_security(self, union): + noinvariant = set() for select in union.children[:]: for subquery in select.with_: - self._insert_security(subquery.query, noinvariant) + self._insert_security(subquery.query) localchecks, restricted = self._check_permissions(select) if any(localchecks): - rewrite = self.session.rql_rewriter.rewrite - nbtrees = len(localchecks) - myunion = union - # transform in subquery when len(localchecks)>1 and groups - if nbtrees > 1 and (select.orderby or select.groupby or - select.having or select.has_aggregat or - select.distinct or - select.limit or select.offset): - newselect = Select() - # only select variables in subqueries - origselection = select.selection - select.select_only_variables() - select.has_aggregat = False - # create subquery first so correct node are used on copy - # (eg ColumnAlias instead of Variable) - aliases = [VariableRef(newselect.get_variable(vref.name, i)) - for i, vref in enumerate(select.selection)] - selected = set(vref.name for vref in aliases) - # now copy original selection and groups - for term in origselection: - newselect.append_selected(term.copy(newselect)) - if select.orderby: - sortterms = [] - for sortterm in select.orderby: - sortterms.append(sortterm.copy(newselect)) - for fnode in sortterm.get_nodes(Function): - if fnode.name == 'FTIRANK': - # we've to fetch the has_text relation as well - var = fnode.children[0].variable - rel = iter(var.stinfo['ftirels']).next() - assert not rel.ored(), 'unsupported' - newselect.add_restriction(rel.copy(newselect)) - # remove relation from the orig select and - # cleanup variable stinfo - rel.parent.remove(rel) - var.stinfo['ftirels'].remove(rel) - var.stinfo['relations'].remove(rel) - # XXX not properly re-annotated after security insertion? - newvar = newselect.get_variable(var.name) - newvar.stinfo.setdefault('ftirels', set()).add(rel) - newvar.stinfo.setdefault('relations', set()).add(rel) - newselect.set_orderby(sortterms) - _expand_selection(select.orderby, selected, aliases, select, newselect) - select.orderby = () # XXX dereference? - if select.groupby: - newselect.set_groupby([g.copy(newselect) for g in select.groupby]) - _expand_selection(select.groupby, selected, aliases, select, newselect) - select.groupby = () # XXX dereference? - if select.having: - newselect.set_having([g.copy(newselect) for g in select.having]) - _expand_selection(select.having, selected, aliases, select, newselect) - select.having = () # XXX dereference? - if select.limit: - newselect.limit = select.limit - select.limit = None - if select.offset: - newselect.offset = select.offset - select.offset = 0 - myunion = Union() - newselect.set_with([SubQuery(aliases, myunion)], check=False) - newselect.distinct = select.distinct - solutions = [sol.copy() for sol in select.solutions] - cleanup_solutions(newselect, solutions) - newselect.set_possible_types(solutions) - # if some solutions doesn't need rewriting, insert original - # select as first union subquery - if () in localchecks: - myunion.append(select) - # we're done, replace original select by the new select with - # subqueries (more added in the loop below) - union.replace(select, newselect) - elif not () in localchecks: - union.remove(select) - for lcheckdef, lchecksolutions in localchecks.iteritems(): - if not lcheckdef: - continue - myrqlst = select.copy(solutions=lchecksolutions) - myunion.append(myrqlst) - # in-place rewrite + annotation / simplification - lcheckdef = [({var: 'X'}, rqlexprs) for var, rqlexprs in lcheckdef] - rewrite(myrqlst, lcheckdef, lchecksolutions, self.args) - add_noinvariant(noinvariant, restricted, myrqlst, nbtrees) - if () in localchecks: - select.set_possible_types(localchecks[()]) - add_types_restriction(self.schema, select) - add_noinvariant(noinvariant, restricted, select, nbtrees) - self.rqlhelper.annotate(union) + self.session.rql_rewriter.insert_local_checks( + select, self.args, localchecks, restricted, noinvariant) + return noinvariant def _check_permissions(self, rqlst): - """return a dict defining "local checks", e.g. RQLExpression defined in - the schema that should be inserted in the original query - - solutions where a variable has a type which the user can't definitly read - are removed, else if the user may read it (eg if an rql expression is - defined for the "read" permission of the related type), the local checks - dict for the solution is updated + """Return a dict defining "local checks", i.e. RQLExpression defined in + the schema that should be inserted in the original query, together with + a set of variable names which requires some security to be inserted. - return a dict with entries for each different local check necessary, - with associated solutions as value. A local check is defined by a list - of 2-uple, with variable name as first item and the necessary rql - expression as second item for each variable which has to be checked. - So solutions which don't require local checks will be associated to - the empty tuple key. + Solutions where a variable has a type which the user can't definitly + read are removed, else if the user *may* read it (i.e. if an rql + expression is defined for the "read" permission of the related type), + the local checks dict is updated. - note: rqlst should not have been simplified at this point + The local checks dict has entries for each different local check + necessary, with associated solutions as value, a local check being + defined by a list of 2-uple (variable name, rql expressions) for each + variable which has to be checked. Solutions which don't require local + checks will be associated to the empty tuple key. + + Note rqlst should not have been simplified at this point. """ session = self.session msgs = [] - neweids = session.transaction_data.get('neweids', ()) + # dict(varname: eid), allowing to check rql expression for variables + # which have a known eid varkwargs = {} if not session.transaction_data.get('security-rqlst-cache'): for var in rqlst.defined_vars.itervalues(): @@ -414,20 +300,27 @@ rqlexprs = localcheck.pop(varname) except KeyError: continue - if eid in neweids: + # if entity has been added in the current transaction, the + # user can read it whatever rql expressions are associated + # to its type + if session.added_in_transaction(eid): continue for rqlexpr in rqlexprs: if rqlexpr.check(session, eid): break else: raise Unauthorized('No read acces on %r with eid %i.' % (var, eid)) + # mark variables protected by an rql expression restricted_vars.update(localcheck) - localchecks.setdefault(tuple(localcheck.iteritems()), []).append(solution) + # turn local check into a dict key + localcheck = tuple(sorted(localcheck.iteritems())) + localchecks.setdefault(localcheck, []).append(solution) # raise Unautorized exception if the user can't access to any solution if not newsolutions: raise Unauthorized('\n'.join(msgs)) + # if there is some message, solutions have been modified and must be + # reconsidered by the syntax treee if msgs: - # (else solutions have not been modified) rqlst.set_possible_types(newsolutions) return localchecks, restricted_vars @@ -728,7 +621,7 @@ if args: # different SQL generated when some argument is None or not (IS # NULL). This should be considered when computing sql cache key - cachekey += tuple(sorted([k for k,v in args.iteritems() + cachekey += tuple(sorted([k for k, v in args.iteritems() if v is None])) # make an execution plan plan = self.plan_factory(rqlst, args, session) diff -r e47e192ea0d9 -r c05652b108ce server/sources/rql2sql.py --- a/server/sources/rql2sql.py Fri Jul 12 10:39:01 2013 +0200 +++ b/server/sources/rql2sql.py Mon Jul 15 10:59:34 2013 +0200 @@ -1,4 +1,4 @@ -# copyright 2003-2012 LOGILAB S.A. (Paris, FRANCE), all rights reserved. +# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved. # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr # # This file is part of CubicWeb. @@ -62,8 +62,8 @@ Not, Comparison, ColumnAlias, Relation, SubQuery, Exists) from cubicweb import QueryError +from cubicweb.rqlrewrite import cleanup_solutions from cubicweb.server.sqlutils import SQL_PREFIX -from cubicweb.server.utils import cleanup_solutions ColumnAlias._q_invariant = False # avoid to check for ColumnAlias / Variable diff -r e47e192ea0d9 -r c05652b108ce server/utils.py --- a/server/utils.py Fri Jul 12 10:39:01 2013 +0200 +++ b/server/utils.py Mon Jul 15 10:59:34 2013 +0200 @@ -1,4 +1,4 @@ -# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved. +# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved. # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr # # This file is part of CubicWeb. @@ -91,13 +91,6 @@ return rloop(seqin, []) -def cleanup_solutions(rqlst, solutions): - for sol in solutions: - for vname in list(sol): - if not (vname in rqlst.defined_vars or vname in rqlst.aliases): - del sol[vname] - - def eschema_eid(session, eschema): """get eid of the CWEType entity for the given yams type. You should use this because when schema has been loaded from the file-system, not from the