# HG changeset patch # User Sylvain Thénault # Date 1253103871 -7200 # Node ID 8604a15995d11ea172f913ef5a23c8b103df57a1 # Parent 1ceac4cd4fb78d35839f76a46f7e4b79a746ca40 refactor so that rql rewriter may be used outside the server. Enhance it to be usable for RRQLExpression as well diff -r 1ceac4cd4fb7 -r 8604a15995d1 cwvreg.py --- a/cwvreg.py Wed Sep 16 14:17:12 2009 +0200 +++ b/cwvreg.py Wed Sep 16 14:24:31 2009 +0200 @@ -291,8 +291,7 @@ def set_schema(self, schema): """set instance'schema and load application objects""" - self.schema = schema - clear_cache(self, 'rqlhelper') + self._set_schema(schema) # now we can load application's web objects searchpath = self.config.vregistry_path() self.reset(searchpath, force_reload=False) @@ -303,6 +302,11 @@ etype = str(etype) self.case_insensitive_etypes[etype.lower()] = etype + def _set_schema(self, schema): + """set instance'schema""" + self.schema = schema + clear_cache(self, 'rqlhelper') + def update_schema(self, schema): """update .schema attribute on registered objects, necessary for some tests @@ -386,16 +390,7 @@ # objects on automatic reloading self._needs_iface.clear() - def parse(self, session, rql, args=None): - rqlst = self.rqlhelper.parse(rql) - def type_from_eid(eid, session=session): - return session.describe(eid)[0] - try: - self.rqlhelper.compute_solutions(rqlst, {'eid': type_from_eid}, args) - except UnknownEid: - for select in rqlst.children: - select.solutions = [] - return rqlst + # rql parsing utilities #################################################### @property @cached @@ -403,38 +398,19 @@ return RQLHelper(self.schema, special_relations={'eid': 'uid', 'has_text': 'fti'}) - - @deprecated('use vreg["etypes"].etype_class(etype)') - def etype_class(self, etype): - return self["etypes"].etype_class(etype) - - @deprecated('use vreg["views"].main_template(*args, **kwargs)') - def main_template(self, req, oid='main-template', **context): - return self["views"].main_template(req, oid, **context) - - @deprecated('use vreg[registry].possible_vobjects(*args, **kwargs)') - def possible_vobjects(self, registry, *args, **kwargs): - return self[registry].possible_vobjects(*args, **kwargs) + def solutions(self, req, rqlst, args): + def type_from_eid(eid, req=req): + return req.describe(eid)[0] + self.rqlhelper.compute_solutions(rqlst, {'eid': type_from_eid}, args) - @deprecated('use vreg["actions"].possible_actions(*args, **kwargs)') - def possible_actions(self, req, rset=None, **kwargs): - return self["actions"].possible_actions(req, rest=rset, **kwargs) - - @deprecated("use vreg['boxes'].select_object(...)") - def select_box(self, oid, *args, **kwargs): - return self['boxes'].select_object(oid, *args, **kwargs) - - @deprecated("use vreg['components'].select_object(...)") - def select_component(self, cid, *args, **kwargs): - return self['components'].select_object(cid, *args, **kwargs) - - @deprecated("use vreg['actions'].select_object(...)") - def select_action(self, oid, *args, **kwargs): - return self['actions'].select_object(oid, *args, **kwargs) - - @deprecated("use vreg['views'].select(...)") - def select_view(self, __vid, req, rset=None, **kwargs): - return self['views'].select(__vid, req, rset=rset, **kwargs) + def parse(self, req, rql, args=None): + rqlst = self.rqlhelper.parse(rql) + try: + self.solutions(req, rqlst, args) + except UnknownEid: + for select in rqlst.children: + select.solutions = [] + return rqlst # properties handling ##################################################### @@ -507,6 +483,40 @@ self.warning('%s (you should probably delete that property ' 'from the database)', ex) + # deprecated code #################################################### + + @deprecated('[3.4] use vreg["etypes"].etype_class(etype)') + def etype_class(self, etype): + return self["etypes"].etype_class(etype) + + @deprecated('[3.4] use vreg["views"].main_template(*args, **kwargs)') + def main_template(self, req, oid='main-template', **context): + return self["views"].main_template(req, oid, **context) + + @deprecated('[3.4] use vreg[registry].possible_vobjects(*args, **kwargs)') + def possible_vobjects(self, registry, *args, **kwargs): + return self[registry].possible_vobjects(*args, **kwargs) + + @deprecated('[3.4] use vreg["actions"].possible_actions(*args, **kwargs)') + def possible_actions(self, req, rset=None, **kwargs): + return self["actions"].possible_actions(req, rest=rset, **kwargs) + + @deprecated('[3.4] use vreg["boxes"].select_object(...)') + def select_box(self, oid, *args, **kwargs): + return self['boxes'].select_object(oid, *args, **kwargs) + + @deprecated('[3.4] use vreg["components"].select_object(...)') + def select_component(self, cid, *args, **kwargs): + return self['components'].select_object(cid, *args, **kwargs) + + @deprecated('[3.4] use vreg["actions"].select_object(...)') + def select_action(self, oid, *args, **kwargs): + return self['actions'].select_object(oid, *args, **kwargs) + + @deprecated('[3.4] use vreg["views"].select(...)') + def select_view(self, __vid, req, rset=None, **kwargs): + return self['views'].select(__vid, req, rset=rset, **kwargs) + from datetime import datetime, date, time, timedelta diff -r 1ceac4cd4fb7 -r 8604a15995d1 devtools/fake.py --- a/devtools/fake.py Wed Sep 16 14:17:12 2009 +0200 +++ b/devtools/fake.py Wed Sep 16 14:24:31 2009 +0200 @@ -13,6 +13,7 @@ from indexer import get_indexer from cubicweb import RequestSessionMixIn +from cubicweb.cwvreg import CubicWebVRegistry from cubicweb.web.request import CubicWebRequestBase from cubicweb.devtools import BASE_URL, BaseApptestConfiguration @@ -67,7 +68,7 @@ def __init__(self, *args, **kwargs): if not (args or 'vreg' in kwargs): - kwargs['vreg'] = FakeVReg() + kwargs['vreg'] = CubicWebVRegistry(FakeConfig(), initlog=False) kwargs['https'] = False self._url = kwargs.pop('url', 'view?rql=Blop&vid=blop') super(FakeRequest, self).__init__(*args, **kwargs) @@ -177,7 +178,7 @@ class FakeSession(RequestSessionMixIn): def __init__(self, repo=None, user=None): self.repo = repo - self.vreg = getattr(self.repo, 'vreg', FakeVReg()) + self.vreg = getattr(self.repo, 'vreg', CubicWebVRegistry(FakeConfig(), initlog=False)) self.pool = FakePool() self.user = user or FakeUser() self.is_internal_session = False @@ -210,8 +211,9 @@ self.eids = {} self._count = 0 self.schema = schema - self.vreg = vreg or FakeVReg() self.config = config or FakeConfig() + self.vreg = vreg or CubicWebVRegistry(self.config, initlog=False) + self.vreg.schema = schema def internal_session(self): return FakeSession(self) diff -r 1ceac4cd4fb7 -r 8604a15995d1 devtools/repotest.py --- a/devtools/repotest.py Wed Sep 16 14:17:12 2009 +0200 +++ b/devtools/repotest.py Wed Sep 16 14:24:31 2009 +0200 @@ -108,9 +108,10 @@ schema = None # set this in concret test def setUp(self): + self.repo = FakeRepo(self.schema) self.rqlhelper = RQLHelper(self.schema, special_relations={'eid': 'uid', 'has_text': 'fti'}) - self.qhelper = QuerierHelper(FakeRepo(self.schema), self.schema) + self.qhelper = QuerierHelper(self.repo, self.schema) ExecutionPlan._check_permissions = _dummy_check_permissions rqlannotation._select_principal = _select_principal @@ -129,7 +130,7 @@ #print '********* solutions', solutions self.rqlhelper.simplify(union) #print '********* simplified', union.as_string() - plan = self.qhelper.plan_factory(union, {}, FakeSession()) + plan = self.qhelper.plan_factory(union, {}, FakeSession(self.repo)) plan.preprocess(union) for select in union.children: select.solutions.sort() @@ -167,7 +168,7 @@ set_debug(debug) def _rqlhelper(self): - rqlhelper = self.o._rqlhelper + rqlhelper = self.repo.vreg.rqlhelper # reset uid_func so it don't try to get type from eids rqlhelper._analyser.uid_func = None rqlhelper._analyser.uid_func_mapping = {} @@ -241,7 +242,7 @@ rqlst = self.o.parse(rql, annotate=True) self.o.solutions(self.session, rqlst, kwargs) if rqlst.TYPE == 'select': - self.o._rqlhelper.annotate(rqlst) + self.repo.vreg.rqlhelper.annotate(rqlst) for select in rqlst.children: select.solutions.sort() else: @@ -251,7 +252,7 @@ # monkey patch some methods to get predicatable results ####################### -from cubicweb.server.rqlrewrite import RQLRewriter +from cubicweb.rqlrewrite import RQLRewriter _orig_insert_snippets = RQLRewriter.insert_snippets _orig_build_variantes = RQLRewriter.build_variantes diff -r 1ceac4cd4fb7 -r 8604a15995d1 goa/gaesource.py --- a/goa/gaesource.py Wed Sep 16 14:17:12 2009 +0200 +++ b/goa/gaesource.py Wed Sep 16 14:24:31 2009 +0200 @@ -149,7 +149,7 @@ # ISource interface ####################################################### def compile_rql(self, rql): - rqlst = self.repo.querier._rqlhelper.parse(rql) + rqlst = self.repo.vreg.parse(rql) rqlst.restricted_vars = () rqlst.children[0].solutions = self._sols return rqlst diff -r 1ceac4cd4fb7 -r 8604a15995d1 goa/goactl.py --- a/goa/goactl.py Wed Sep 16 14:17:12 2009 +0200 +++ b/goa/goactl.py Wed Sep 16 14:24:31 2009 +0200 @@ -54,6 +54,7 @@ 'cwconfig.py', 'entity.py', 'interfaces.py', + 'rqlrewrite.py', 'rset.py', 'schema.py', 'schemaviewer.py', @@ -78,7 +79,6 @@ 'server/pool.py', 'server/querier.py', 'server/repository.py', - 'server/rqlrewrite.py', 'server/securityhooks.py', 'server/session.py', 'server/serverconfig.py', diff -r 1ceac4cd4fb7 -r 8604a15995d1 rqlrewrite.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/rqlrewrite.py Wed Sep 16 14:24:31 2009 +0200 @@ -0,0 +1,480 @@ +"""RQL rewriting utilities : insert rql expression snippets into rql syntax +tree. + +This is used for instance for read security checking in the repository. + +:organization: Logilab +:copyright: 2007-2009 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. +:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr +:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses +""" +__docformat__ = "restructuredtext en" + +from rql import nodes as n, stmts, TypeResolverException + +from logilab.common.compat import any + +from cubicweb import Unauthorized, server, typed_eid +from cubicweb.server.ssplanner import add_types_restriction + + +def remove_solutions(origsolutions, solutions, defined): + """when a rqlst has been generated from another by introducing security + assertions, this method returns solutions which are contained in orig + solutions + """ + newsolutions = [] + for origsol in origsolutions: + for newsol in solutions[:]: + for var, etype in origsol.items(): + try: + if newsol[var] != etype: + try: + defined[var].stinfo['possibletypes'].remove(newsol[var]) + except KeyError: + pass + break + except KeyError: + # variable has been rewritten + continue + else: + newsolutions.append(newsol) + solutions.remove(newsol) + return newsolutions + + +class Unsupported(Exception): pass + + +class RQLRewriter(object): + """insert some rql snippets into another rql syntax tree + + this class *isn't thread safe* + """ + + def __init__(self, session): + self.session = session + vreg = session.vreg + self.schema = vreg.schema + self.annotate = vreg.rqlhelper.annotate + self._compute_solutions = vreg.solutions + + def compute_solutions(self): + self.annotate(self.select) + try: + self._compute_solutions(self.session, self.select, self.kwargs) + except TypeResolverException: + raise Unsupported(str(self.select)) + if len(self.select.solutions) < len(self.solutions): + raise Unsupported() + + def rewrite(self, select, snippets, solutions, kwargs): + """ + snippets: (varmap, list of rql expression) + with varmap a *tuple* (select var, snippet var) + """ + if server.DEBUG: + print '---- rewrite', select, snippets, solutions + self.select = self.insert_scope = select + self.solutions = solutions + self.kwargs = kwargs + self.u_varname = None + self.removing_ambiguity = False + self.exists_snippet = {} + self.pending_keys = [] + # we have to annotate the rqlst before inserting snippets, even though + # we'll have to redo it latter + self.annotate(select) + self.insert_snippets(snippets) + if not self.exists_snippet and self.u_varname: + # U has been inserted than cancelled, cleanup + select.undefine_variable(select.defined_vars[self.u_varname]) + # clean solutions according to initial solutions + newsolutions = remove_solutions(solutions, select.solutions, + select.defined_vars) + assert len(newsolutions) >= len(solutions), ( + 'rewritten rql %s has lost some solutions, there is probably ' + 'something wrong in your schema permission (for instance using a ' + 'RQLExpression which insert a relation which doesn\'t exists in ' + 'the schema)\nOrig solutions: %s\nnew solutions: %s' % ( + select, solutions, newsolutions)) + if len(newsolutions) > len(solutions): + newsolutions = self.remove_ambiguities(snippets, newsolutions) + select.solutions = newsolutions + add_types_restriction(self.schema, select) + if server.DEBUG: + print '---- rewriten', select + + def insert_snippets(self, snippets, varexistsmap=None): + self.rewritten = {} + for varmap, rqlexprs in snippets: + if varexistsmap is not None and not varmap in varexistsmap: + continue + self.varmap = varmap + selectvar, snippetvar = varmap + assert snippetvar in 'SOX' + self.revvarmap = {snippetvar: selectvar} + self.varinfo = vi = {} + try: + vi['const'] = typed_eid(selectvar) # XXX gae + vi['rhs_rels'] = vi['lhs_rels'] = {} + except ValueError: + vi['stinfo'] = sti = self.select.defined_vars[selectvar].stinfo + if varexistsmap is None: + vi['rhs_rels'] = dict( (r.r_type, r) for r in sti['rhsrelations']) + vi['lhs_rels'] = dict( (r.r_type, r) for r in sti['relations'] + if not r in sti['rhsrelations']) + else: + vi['rhs_rels'] = vi['lhs_rels'] = {} + parent = None + inserted = False + for rqlexpr in rqlexprs: + self.current_expr = rqlexpr + if varexistsmap is None: + try: + new = self.insert_snippet(varmap, rqlexpr.snippet_rqlst, parent) + except Unsupported: + import traceback + traceback.print_exc() + continue + inserted = True + if new is not None: + self.exists_snippet[rqlexpr] = new + parent = parent or new + else: + # called to reintroduce snippet due to ambiguity creation, + # so skip snippets which are not introducing this ambiguity + exists = varexistsmap[varmap] + if self.exists_snippet[rqlexpr] is exists: + self.insert_snippet(varmap, rqlexpr.snippet_rqlst, exists) + if varexistsmap is None and not inserted: + # no rql expression found matching rql solutions. User has no access right + raise Unauthorized(str((varmap, str(self.select), [expr.expression for expr in rqlexprs]))) + + def insert_snippet(self, varmap, snippetrqlst, parent=None): + new = snippetrqlst.where.accept(self) + if new is not None: + if self.varinfo.get('stinfo', {}).get('optrelations'): + assert parent is None + self.insert_scope = self.snippet_subquery(varmap, new) + self.insert_pending() + self.insert_scope = self.select + return + new = n.Exists(new) + if parent is None: + self.insert_scope.add_restriction(new) + else: + grandpa = parent.parent + or_ = n.Or(parent, new) + grandpa.replace(parent, or_) + if not self.removing_ambiguity: + try: + self.compute_solutions() + except Unsupported: + # some solutions have been lost, can't apply this rql expr + if parent is None: + self.select.remove_node(new, undefine=True) + else: + parent.parent.replace(or_, or_.children[0]) + self._cleanup_inserted(new) + raise + else: + self.insert_scope = new + self.insert_pending() + self.insert_scope = self.select + return new + self.insert_pending() + + def insert_pending(self): + """pending_keys hold variable referenced by U has__permission X + relation. + + Once the snippet introducing this has been inserted and solutions + recomputed, we have to insert snippet defined for of entity + types taken by X + """ + while self.pending_keys: + key, action = self.pending_keys.pop() + try: + varname = self.rewritten[key] + except KeyError: + try: + varname = self.revvarmap[key[-1]] + except KeyError: + # variable isn't used anywhere else, we can't insert security + raise Unauthorized() + ptypes = self.select.defined_vars[varname].stinfo['possibletypes'] + if len(ptypes) > 1: + # XXX dunno how to handle this + self.session.error( + 'cant check security of %s, ambigous type for %s in %s', + self.select, varname, key[0]) # key[0] == the rql expression + raise Unauthorized() + etype = iter(ptypes).next() + eschema = self.schema.eschema(etype) + if not eschema.has_perm(self.session, action): + rqlexprs = eschema.get_rqlexprs(action) + if not rqlexprs: + raise Unauthorised() + self.insert_snippets([((varname, 'X'), rqlexprs)]) + + def snippet_subquery(self, varmap, transformedsnippet): + """introduce the given snippet in a subquery""" + subselect = stmts.Select() + selectvar, snippetvar = varmap + subselect.append_selected(n.VariableRef( + subselect.get_variable(selectvar))) + aliases = [selectvar] + subselect.add_restriction(transformedsnippet.copy(subselect)) + stinfo = self.varinfo['stinfo'] + for rel in stinfo['relations']: + rschema = self.schema.rschema(rel.r_type) + if rschema.is_final() or (rschema.inlined and + not rel in stinfo['rhsrelations']): + self.select.remove_node(rel) + rel.children[0].name = selectvar + subselect.add_restriction(rel.copy(subselect)) + for vref in rel.children[1].iget_nodes(n.VariableRef): + subselect.append_selected(vref.copy(subselect)) + aliases.append(vref.name) + if self.u_varname: + # generate an identifier for the substitution + argname = subselect.allocate_varname() + while argname in self.kwargs: + argname = subselect.allocate_varname() + subselect.add_constant_restriction(subselect.get_variable(self.u_varname), + 'eid', unicode(argname), 'Substitute') + self.kwargs[argname] = self.session.user.eid + add_types_restriction(self.schema, subselect, subselect, + solutions=self.solutions) + myunion = stmts.Union() + myunion.append(subselect) + aliases = [n.VariableRef(self.select.get_variable(name, i)) + for i, name in enumerate(aliases)] + self.select.add_subquery(n.SubQuery(aliases, myunion), check=False) + self._cleanup_inserted(transformedsnippet) + try: + self.compute_solutions() + except Unsupported: + # some solutions have been lost, can't apply this rql expr + self.select.remove_subquery(new, undefine=True) + raise + return subselect + + def remove_ambiguities(self, snippets, newsolutions): + # the snippet has introduced some ambiguities, we have to resolve them + # "manually" + variantes = self.build_variantes(newsolutions) + # insert "is" where necessary + varexistsmap = {} + self.removing_ambiguity = True + for (erqlexpr, varmap, oldvarname), etype in variantes[0].iteritems(): + varname = self.rewritten[(erqlexpr, varmap, oldvarname)] + var = self.select.defined_vars[varname] + exists = var.references()[0].scope + exists.add_constant_restriction(var, 'is', etype, 'etype') + varexistsmap[varmap] = exists + # insert ORED exists where necessary + for variante in variantes[1:]: + self.insert_snippets(snippets, varexistsmap) + for key, etype in variante.iteritems(): + varname = self.rewritten[key] + try: + var = self.select.defined_vars[varname] + except KeyError: + # not a newly inserted variable + continue + exists = var.references()[0].scope + exists.add_constant_restriction(var, 'is', etype, 'etype') + # recompute solutions + #select.annotated = False # avoid assertion error + self.compute_solutions() + # clean solutions according to initial solutions + return remove_solutions(self.solutions, self.select.solutions, + self.select.defined_vars) + + def build_variantes(self, newsolutions): + variantes = set() + for sol in newsolutions: + variante = [] + for key, newvar in self.rewritten.iteritems(): + variante.append( (key, sol[newvar]) ) + variantes.add(tuple(variante)) + # rebuild variantes as dict + variantes = [dict(variante) for variante in variantes] + # remove variable which have always the same type + for key in self.rewritten: + it = iter(variantes) + etype = it.next()[key] + for variante in it: + if variante[key] != etype: + break + else: + for variante in variantes: + del variante[key] + return variantes + + def _cleanup_inserted(self, node): + # cleanup inserted variable references + for vref in node.iget_nodes(n.VariableRef): + vref.unregister_reference() + if not vref.variable.stinfo['references']: + # no more references, undefine the variable + del self.select.defined_vars[vref.name] + + def _may_be_shared(self, relation, target, searchedvarname): + """return True if the snippet relation can be skipped to use a relation + from the original query + """ + # if cardinality is in '?1', we can ignore the relation and use variable + # from the original query + rschema = self.schema.rschema(relation.r_type) + if target == 'object': + cardindex = 0 + ttypes_func = rschema.objects + rprop = rschema.rproperty + else: # target == 'subject': + cardindex = 1 + ttypes_func = rschema.subjects + rprop = lambda x, y, z: rschema.rproperty(y, x, z) + for etype in self.varinfo['stinfo']['possibletypes']: + for ttype in ttypes_func(etype): + if rprop(etype, ttype, 'cardinality')[cardindex] in '+*': + return False + return True + + def _use_outer_term(self, snippet_varname, term): + key = (self.current_expr, self.varmap, snippet_varname) + if key in self.rewritten: + insertedvar = self.select.defined_vars.pop(self.rewritten[key]) + for inserted_vref in insertedvar.references(): + inserted_vref.parent.replace(inserted_vref, term.copy(self.select)) + self.rewritten[key] = term.name + + def _get_varname_or_term(self, vname): + if vname == 'U': + if self.u_varname is None: + select = self.select + self.u_varname = select.allocate_varname() + # generate an identifier for the substitution + argname = select.allocate_varname() + while argname in self.kwargs: + argname = select.allocate_varname() + # insert "U eid %(u)s" + var = select.get_variable(self.u_varname) + select.add_constant_restriction(select.get_variable(self.u_varname), + 'eid', unicode(argname), 'Substitute') + self.kwargs[argname] = self.session.user.eid + return self.u_varname + key = (self.current_expr, self.varmap, vname) + try: + return self.rewritten[key] + except KeyError: + self.rewritten[key] = newvname = self.select.allocate_varname() + return newvname + + # visitor methods ########################################################## + + def _visit_binary(self, node, cls): + newnode = cls() + for c in node.children: + new = c.accept(self) + if new is None: + continue + newnode.append(new) + if len(newnode.children) == 0: + return None + if len(newnode.children) == 1: + return newnode.children[0] + return newnode + + def _visit_unary(self, node, cls): + newc = node.children[0].accept(self) + if newc is None: + return None + newnode = cls() + newnode.append(newc) + return newnode + + def visit_and(self, node): + return self._visit_binary(node, n.And) + + def visit_or(self, node): + return self._visit_binary(node, n.Or) + + def visit_not(self, node): + return self._visit_unary(node, n.Not) + + def visit_exists(self, node): + return self._visit_unary(node, n.Exists) + + def visit_relation(self, node): + lhs, rhs = node.get_variable_parts() + if node.r_type in ('has_add_permission', 'has_update_permission', + 'has_delete_permission', 'has_read_permission'): + assert lhs.name == 'U' + action = node.r_type.split('_')[1] + key = (self.current_expr, self.varmap, rhs.name) + self.pending_keys.append( (key, action) ) + return + if lhs.name in self.revvarmap: + # on lhs + # see if we can reuse this relation + rels = self.varinfo['lhs_rels'] + if (node.r_type in rels and isinstance(rhs, n.VariableRef) + and rhs.name != 'U' and not rels[node.r_type].neged(strict=True) + and self._may_be_shared(node, 'object', lhs.name)): + # ok, can share variable + term = rels[node.r_type].children[1].children[0] + self._use_outer_term(rhs.name, term) + return + elif isinstance(rhs, n.VariableRef) and rhs.name in self.revvarmap and lhs.name != 'U': + # on rhs + # see if we can reuse this relation + rels = self.varinfo['rhs_rels'] + if (node.r_type in rels and not rels[node.r_type].neged(strict=True) + and self._may_be_shared(node, 'subject', rhs.name)): + # ok, can share variable + term = rels[node.r_type].children[0] + self._use_outer_term(lhs.name, term) + return + rel = n.Relation(node.r_type, node.optional) + for c in node.children: + rel.append(c.accept(self)) + return rel + + def visit_comparison(self, node): + cmp_ = n.Comparison(node.operator) + for c in node.children: + cmp_.append(c.accept(self)) + return cmp_ + + def visit_mathexpression(self, node): + cmp_ = n.MathExpression(node.operator) + for c in cmp.children: + cmp_.append(c.accept(self)) + return cmp_ + + def visit_function(self, node): + """generate filter name for a function""" + function_ = n.Function(node.name) + for c in node.children: + function_.append(c.accept(self)) + return function_ + + def visit_constant(self, node): + """generate filter name for a constant""" + return n.Constant(node.value, node.type) + + def visit_variableref(self, node): + """get the sql name for a variable reference""" + if node.name in self.revvarmap: + if self.varinfo.get('const') is not None: + return n.Constant(self.varinfo['const'], 'Int') # XXX gae + return n.VariableRef(self.select.get_variable( + self.revvarmap[node.name])) + vname_or_term = self._get_varname_or_term(node.name) + if isinstance(vname_or_term, basestring): + return n.VariableRef(self.select.get_variable(vname_or_term)) + # shared term + return vname_or_term.copy(self.select) diff -r 1ceac4cd4fb7 -r 8604a15995d1 schema.py --- a/schema.py Wed Sep 16 14:17:12 2009 +0200 +++ b/schema.py Wed Sep 16 14:24:31 2009 +0200 @@ -642,6 +642,8 @@ if len(self.rqlst.defined_vars[mainvar].references()) <= 2: _LOGGER.warn('You did not use the %s variable in your RQL ' 'expression %s', mainvar, self) + # syntax tree used by read security (inserted in queries when necessary + self.snippet_rqlst = parse(self.minimal_rql, print_errors=False).children[0] def __str__(self): return self.full_rql @@ -767,8 +769,6 @@ class ERQLExpression(RQLExpression): def __init__(self, expression, mainvars=None, eid=None): RQLExpression.__init__(self, expression, mainvars or 'X', eid) - # syntax tree used by read security (inserted in queries when necessary - self.snippet_rqlst = parse(self.minimal_rql, print_errors=False).children[0] @property def full_rql(self): diff -r 1ceac4cd4fb7 -r 8604a15995d1 server/querier.py --- a/server/querier.py Wed Sep 16 14:17:12 2009 +0200 +++ b/server/querier.py Wed Sep 16 14:24:31 2009 +0200 @@ -138,8 +138,8 @@ # various resource accesors self.querier = querier self.schema = querier.schema - self.rqlhelper = querier._rqlhelper self.sqlannotate = querier.sqlgen_annotate + self.rqlhelper = session.vreg.rqlhelper def annotate_rqlst(self): if not self.rqlst.annotated: @@ -265,6 +265,8 @@ myrqlst = select.copy(solutions=lchecksolutions) myunion.append(myrqlst) # in-place rewrite + annotation / simplification + lcheckdef = [((varmap, 'X'), rqlexprs) + for varmap, rqlexprs in lcheckdef] rewrite(myrqlst, lcheckdef, lchecksolutions, self.args) noinvariant.update(noinvariant_vars(restricted, myrqlst, nbtrees)) if () in localchecks: @@ -524,37 +526,33 @@ def set_schema(self, schema): self.schema = schema + repo = self._repo # rql parsing / analysing helper - self._rqlhelper = RQLHelper(schema, special_relations={'eid': 'uid', - 'has_text': 'fti'}) - self._rql_cache = Cache(self._repo.config['rql-cache-size']) + self.solutions = repo.vreg.solutions + self._rql_cache = Cache(repo.config['rql-cache-size']) self.cache_hit, self.cache_miss = 0, 0 # rql planner # note: don't use repo.sources, may not be built yet, and also "admin" # isn't an actual source - if len([uri for uri in self._repo.config.sources() if uri != 'admin']) < 2: + rqlhelper = repo.vreg.rqlhelper + self._parse = rqlhelper.parse + self._annotate = rqlhelper.annotate + if len([uri for uri in repo.config.sources() if uri != 'admin']) < 2: from cubicweb.server.ssplanner import SSPlanner - self._planner = SSPlanner(schema, self._rqlhelper) + self._planner = SSPlanner(schema, rqlhelper) else: from cubicweb.server.msplanner import MSPlanner - self._planner = MSPlanner(schema, self._rqlhelper) + self._planner = MSPlanner(schema, rqlhelper) # sql generation annotator self.sqlgen_annotate = SQLGenAnnotator(schema).annotate def parse(self, rql, annotate=False): """return a rql syntax tree for the given rql""" try: - return self._rqlhelper.parse(unicode(rql), annotate=annotate) + return self._parse(unicode(rql), annotate=annotate) except UnicodeError: raise RQLSyntaxError(rql) - def solutions(self, session, rqlst, args): - assert session is not None - def type_from_eid(eid, type_from_eid=self._repo.type_from_eid, - session=session): - return type_from_eid(eid, session) - self._rqlhelper.compute_solutions(rqlst, {'eid': type_from_eid}, args) - def plan_factory(self, rqlst, args, session): """create an execution plan for an INSERT RQL query""" if rqlst.TYPE == 'insert': @@ -642,7 +640,7 @@ # bother modifying it. This is not necessary on write queries since # a new syntax tree is built from them. rqlst = rqlst.copy() - self._rqlhelper.annotate(rqlst) + self._annotate(rqlst) # make an execution plan plan = self.plan_factory(rqlst, args, session) plan.cache_key = cachekey diff -r 1ceac4cd4fb7 -r 8604a15995d1 server/repository.py --- a/server/repository.py Wed Sep 16 14:17:12 2009 +0200 +++ b/server/repository.py Wed Sep 16 14:24:31 2009 +0200 @@ -149,6 +149,7 @@ self._running_threads = [] # initial schema, should be build or replaced latter self.schema = CubicWebSchema(config.appid) + self.vreg.schema = self.schema # until actual schema is loaded... # querier helper, need to be created after sources initialization self.querier = QuerierHelper(self, self.schema) # should we reindex in changes? @@ -192,7 +193,6 @@ config.bootstrap_cubes() self.set_bootstrap_schema(config.load_schema()) # need to load the Any and CWUser entity types - self.vreg.schema = self.schema etdirectory = join(CW_SOFTWARE_ROOT, 'entities') self.vreg.init_registration([etdirectory]) self.vreg.load_file(join(etdirectory, '__init__.py'), @@ -246,15 +246,16 @@ if rebuildinfered: schema.rebuild_infered_relations() self.info('set schema %s %#x', schema.name, id(schema)) - self.debug(', '.join(sorted(str(e) for e in schema.entities()))) + if resetvreg: + # full reload of all appobjects + self.vreg.reset() + self.vreg.set_schema(schema) + else: + self.vreg._set_schema(schema) self.querier.set_schema(schema) for source in self.sources: source.set_schema(schema) self.schema = schema - if resetvreg: - # full reload of all appobjects - self.vreg.reset() - self.vreg.set_schema(schema) self.reset_hooks() def reset_hooks(self): diff -r 1ceac4cd4fb7 -r 8604a15995d1 server/rqlrewrite.py --- a/server/rqlrewrite.py Wed Sep 16 14:17:12 2009 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,396 +0,0 @@ -"""RQL rewriting utilities, used for read security checking - -:organization: Logilab -:copyright: 2007-2009 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. -:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr -:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses -""" - -from rql import nodes, stmts, TypeResolverException -from cubicweb import Unauthorized, server, typed_eid -from cubicweb.server.ssplanner import add_types_restriction - -def remove_solutions(origsolutions, solutions, defined): - """when a rqlst has been generated from another by introducing security - assertions, this method returns solutions which are contained in orig - solutions - """ - newsolutions = [] - for origsol in origsolutions: - for newsol in solutions[:]: - for var, etype in origsol.items(): - try: - if newsol[var] != etype: - try: - defined[var].stinfo['possibletypes'].remove(newsol[var]) - except KeyError: - pass - break - except KeyError: - # variable has been rewritten - continue - else: - newsolutions.append(newsol) - solutions.remove(newsol) - return newsolutions - -class Unsupported(Exception): pass - -class RQLRewriter(object): - """insert some rql snippets into another rql syntax tree""" - def __init__(self, querier, session): - self.session = session - self.annotate = querier._rqlhelper.annotate - self._compute_solutions = querier.solutions - self.schema = querier.schema - - def compute_solutions(self): - self.annotate(self.select) - try: - self._compute_solutions(self.session, self.select, self.kwargs) - except TypeResolverException: - raise Unsupported() - if len(self.select.solutions) < len(self.solutions): - raise Unsupported() - - def rewrite(self, select, snippets, solutions, kwargs): - if server.DEBUG: - print '---- rewrite', select, snippets, solutions - self.select = select - self.solutions = solutions - self.kwargs = kwargs - self.u_varname = None - self.removing_ambiguity = False - self.exists_snippet = {} - # we have to annotate the rqlst before inserting snippets, even though - # we'll have to redo it latter - self.annotate(select) - self.insert_snippets(snippets) - if not self.exists_snippet and self.u_varname: - # U has been inserted than cancelled, cleanup - select.undefine_variable(select.defined_vars[self.u_varname]) - # clean solutions according to initial solutions - newsolutions = remove_solutions(solutions, select.solutions, - select.defined_vars) - assert len(newsolutions) >= len(solutions), \ - 'rewritten rql %s has lost some solutions, there is probably something '\ - 'wrong in your schema permission (for instance using a '\ - 'RQLExpression which insert a relation which doesn\'t exists in '\ - 'the schema)\nOrig solutions: %s\nnew solutions: %s' % ( - select, solutions, newsolutions) - if len(newsolutions) > len(solutions): - # the snippet has introduced some ambiguities, we have to resolve them - # "manually" - variantes = self.build_variantes(newsolutions) - # insert "is" where necessary - varexistsmap = {} - self.removing_ambiguity = True - for (erqlexpr, mainvar, oldvarname), etype in variantes[0].iteritems(): - varname = self.rewritten[(erqlexpr, mainvar, oldvarname)] - var = select.defined_vars[varname] - exists = var.references()[0].scope - exists.add_constant_restriction(var, 'is', etype, 'etype') - varexistsmap[mainvar] = exists - # insert ORED exists where necessary - for variante in variantes[1:]: - self.insert_snippets(snippets, varexistsmap) - for (erqlexpr, mainvar, oldvarname), etype in variante.iteritems(): - varname = self.rewritten[(erqlexpr, mainvar, oldvarname)] - try: - var = select.defined_vars[varname] - except KeyError: - # not a newly inserted variable - continue - exists = var.references()[0].scope - exists.add_constant_restriction(var, 'is', etype, 'etype') - # recompute solutions - #select.annotated = False # avoid assertion error - self.compute_solutions() - # clean solutions according to initial solutions - newsolutions = remove_solutions(solutions, select.solutions, - select.defined_vars) - select.solutions = newsolutions - add_types_restriction(self.schema, select) - if server.DEBUG: - print '---- rewriten', select - - def build_variantes(self, newsolutions): - variantes = set() - for sol in newsolutions: - variante = [] - for (erqlexpr, mainvar, oldvar), newvar in self.rewritten.iteritems(): - variante.append( ((erqlexpr, mainvar, oldvar), sol[newvar]) ) - variantes.add(tuple(variante)) - # rebuild variantes as dict - variantes = [dict(variante) for variante in variantes] - # remove variable which have always the same type - for erqlexpr, mainvar, oldvar in self.rewritten: - it = iter(variantes) - etype = it.next()[(erqlexpr, mainvar, oldvar)] - for variante in it: - if variante[(erqlexpr, mainvar, oldvar)] != etype: - break - else: - for variante in variantes: - del variante[(erqlexpr, mainvar, oldvar)] - return variantes - - def insert_snippets(self, snippets, varexistsmap=None): - self.rewritten = {} - for varname, erqlexprs in snippets: - if varexistsmap is not None and not varname in varexistsmap: - continue - try: - self.const = typed_eid(varname) - self.varname = self.const - self.rhs_rels = self.lhs_rels = {} - except ValueError: - self.varname = varname - self.const = None - self.varstinfo = stinfo = self.select.defined_vars[varname].stinfo - if varexistsmap is None: - self.rhs_rels = dict( (rel.r_type, rel) for rel in stinfo['rhsrelations']) - self.lhs_rels = dict( (rel.r_type, rel) for rel in stinfo['relations'] - if not rel in stinfo['rhsrelations']) - else: - self.rhs_rels = self.lhs_rels = {} - parent = None - inserted = False - for erqlexpr in erqlexprs: - self.current_expr = erqlexpr - if varexistsmap is None: - try: - new = self.insert_snippet(varname, erqlexpr.snippet_rqlst, parent) - except Unsupported: - continue - inserted = True - if new is not None: - self.exists_snippet[erqlexpr] = new - parent = parent or new - else: - # called to reintroduce snippet due to ambiguity creation, - # so skip snippets which are not introducing this ambiguity - exists = varexistsmap[varname] - if self.exists_snippet[erqlexpr] is exists: - self.insert_snippet(varname, erqlexpr.snippet_rqlst, exists) - if varexistsmap is None and not inserted: - # no rql expression found matching rql solutions. User has no access right - raise Unauthorized() - - def insert_snippet(self, varname, snippetrqlst, parent=None): - new = snippetrqlst.where.accept(self) - if new is not None: - try: - var = self.select.defined_vars[varname] - except KeyError: - # not a variable - pass - else: - if var.stinfo['optrelations']: - # use a subquery - subselect = stmts.Select() - subselect.append_selected(nodes.VariableRef(subselect.get_variable(varname))) - subselect.add_restriction(new.copy(subselect)) - aliases = [varname] - for rel in var.stinfo['relations']: - rschema = self.schema.rschema(rel.r_type) - if rschema.is_final() or (rschema.inlined and not rel in var.stinfo['rhsrelations']): - self.select.remove_node(rel) - rel.children[0].name = varname - subselect.add_restriction(rel.copy(subselect)) - for vref in rel.children[1].iget_nodes(nodes.VariableRef): - subselect.append_selected(vref.copy(subselect)) - aliases.append(vref.name) - if self.u_varname: - # generate an identifier for the substitution - argname = subselect.allocate_varname() - while argname in self.kwargs: - argname = subselect.allocate_varname() - subselect.add_constant_restriction(subselect.get_variable(self.u_varname), - 'eid', unicode(argname), 'Substitute') - self.kwargs[argname] = self.session.user.eid - add_types_restriction(self.schema, subselect, subselect, solutions=self.solutions) - assert parent is None - myunion = stmts.Union() - myunion.append(subselect) - aliases = [nodes.VariableRef(self.select.get_variable(name, i)) - for i, name in enumerate(aliases)] - self.select.add_subquery(nodes.SubQuery(aliases, myunion), check=False) - self._cleanup_inserted(new) - try: - self.compute_solutions() - except Unsupported: - # some solutions have been lost, can't apply this rql expr - self.select.remove_subquery(new, undefine=True) - raise - return - new = nodes.Exists(new) - if parent is None: - self.select.add_restriction(new) - else: - grandpa = parent.parent - or_ = nodes.Or(parent, new) - grandpa.replace(parent, or_) - if not self.removing_ambiguity: - try: - self.compute_solutions() - except Unsupported: - # some solutions have been lost, can't apply this rql expr - if parent is None: - self.select.remove_node(new, undefine=True) - else: - parent.parent.replace(or_, or_.children[0]) - self._cleanup_inserted(new) - raise - return new - - def _cleanup_inserted(self, node): - # cleanup inserted variable references - for vref in node.iget_nodes(nodes.VariableRef): - vref.unregister_reference() - if not vref.variable.stinfo['references']: - # no more references, undefine the variable - del self.select.defined_vars[vref.name] - - def _visit_binary(self, node, cls): - newnode = cls() - for c in node.children: - new = c.accept(self) - if new is None: - continue - newnode.append(new) - if len(newnode.children) == 0: - return None - if len(newnode.children) == 1: - return newnode.children[0] - return newnode - - def _visit_unary(self, node, cls): - newc = node.children[0].accept(self) - if newc is None: - return None - newnode = cls() - newnode.append(newc) - return newnode - - def visit_and(self, et): - return self._visit_binary(et, nodes.And) - - def visit_or(self, ou): - return self._visit_binary(ou, nodes.Or) - - def visit_not(self, node): - return self._visit_unary(node, nodes.Not) - - def visit_exists(self, node): - return self._visit_unary(node, nodes.Exists) - - def visit_relation(self, relation): - lhs, rhs = relation.get_variable_parts() - if lhs.name == 'X': - # on lhs - # see if we can reuse this relation - if relation.r_type in self.lhs_rels and isinstance(rhs, nodes.VariableRef) and rhs.name != 'U': - if self._may_be_shared(relation, 'object'): - # ok, can share variable - term = self.lhs_rels[relation.r_type].children[1].children[0] - self._use_outer_term(rhs.name, term) - return - elif isinstance(rhs, nodes.VariableRef) and rhs.name == 'X' and lhs.name != 'U': - # on rhs - # see if we can reuse this relation - if relation.r_type in self.rhs_rels and self._may_be_shared(relation, 'subject'): - # ok, can share variable - term = self.rhs_rels[relation.r_type].children[0] - self._use_outer_term(lhs.name, term) - return - rel = nodes.Relation(relation.r_type, relation.optional) - for c in relation.children: - rel.append(c.accept(self)) - return rel - - def visit_comparison(self, cmp): - cmp_ = nodes.Comparison(cmp.operator) - for c in cmp.children: - cmp_.append(c.accept(self)) - return cmp_ - - def visit_mathexpression(self, mexpr): - cmp_ = nodes.MathExpression(mexpr.operator) - for c in cmp.children: - cmp_.append(c.accept(self)) - return cmp_ - - def visit_function(self, function): - """generate filter name for a function""" - function_ = nodes.Function(function.name) - for c in function.children: - function_.append(c.accept(self)) - return function_ - - def visit_constant(self, constant): - """generate filter name for a constant""" - return nodes.Constant(constant.value, constant.type) - - def visit_variableref(self, vref): - """get the sql name for a variable reference""" - if vref.name == 'X': - if self.const is not None: - return nodes.Constant(self.const, 'Int') - return nodes.VariableRef(self.select.get_variable(self.varname)) - vname_or_term = self._get_varname_or_term(vref.name) - if isinstance(vname_or_term, basestring): - return nodes.VariableRef(self.select.get_variable(vname_or_term)) - # shared term - return vname_or_term.copy(self.select) - - def _may_be_shared(self, relation, target): - """return True if the snippet relation can be skipped to use a relation - from the original query - """ - # if cardinality is in '?1', we can ignore the relation and use variable - # from the original query - rschema = self.schema.rschema(relation.r_type) - if target == 'object': - cardindex = 0 - ttypes_func = rschema.objects - rprop = rschema.rproperty - else: # target == 'subject': - cardindex = 1 - ttypes_func = rschema.subjects - rprop = lambda x, y, z: rschema.rproperty(y, x, z) - for etype in self.varstinfo['possibletypes']: - for ttype in ttypes_func(etype): - if rprop(etype, ttype, 'cardinality')[cardindex] in '+*': - return False - return True - - def _use_outer_term(self, snippet_varname, term): - key = (self.current_expr, self.varname, snippet_varname) - if key in self.rewritten: - insertedvar = self.select.defined_vars.pop(self.rewritten[key]) - for inserted_vref in insertedvar.references(): - inserted_vref.parent.replace(inserted_vref, term.copy(self.select)) - self.rewritten[key] = term - - def _get_varname_or_term(self, vname): - if vname == 'U': - if self.u_varname is None: - select = self.select - self.u_varname = select.allocate_varname() - # generate an identifier for the substitution - argname = select.allocate_varname() - while argname in self.kwargs: - argname = select.allocate_varname() - # insert "U eid %(u)s" - var = select.get_variable(self.u_varname) - select.add_constant_restriction(select.get_variable(self.u_varname), - 'eid', unicode(argname), 'Substitute') - self.kwargs[argname] = self.session.user.eid - return self.u_varname - key = (self.current_expr, self.varname, vname) - try: - return self.rewritten[key] - except KeyError: - self.rewritten[key] = newvname = self.select.allocate_varname() - return newvname diff -r 1ceac4cd4fb7 -r 8604a15995d1 server/session.py --- a/server/session.py Wed Sep 16 14:17:12 2009 +0200 +++ b/server/session.py Wed Sep 16 14:24:31 2009 +0200 @@ -18,7 +18,7 @@ from cubicweb import RequestSessionMixIn, Binary, UnknownEid from cubicweb.dbapi import ConnectionProperties from cubicweb.utils import make_uid -from cubicweb.server.rqlrewrite import RQLRewriter +from cubicweb.rqlrewrite import RQLRewriter ETYPE_PYOBJ_MAP[Binary] = 'Bytes' @@ -543,7 +543,7 @@ try: return self._threaddata._rewriter except AttributeError: - self._threaddata._rewriter = RQLRewriter(self.repo.querier, self) + self._threaddata._rewriter = RQLRewriter(self) return self._threaddata._rewriter def build_description(self, rqlst, args, result): diff -r 1ceac4cd4fb7 -r 8604a15995d1 server/sources/native.py --- a/server/sources/native.py Wed Sep 16 14:17:12 2009 +0200 +++ b/server/sources/native.py Wed Sep 16 14:24:31 2009 +0200 @@ -231,7 +231,7 @@ # ISource interface ####################################################### def compile_rql(self, rql): - rqlst = self.repo.querier._rqlhelper.parse(rql) + rqlst = self.repo.vreg.rqlhelper.parse(rql) rqlst.restricted_vars = () rqlst.children[0].solutions = self._sols self.repo.querier.sqlgen_annotate(rqlst) diff -r 1ceac4cd4fb7 -r 8604a15995d1 server/test/unittest_msplanner.py --- a/server/test/unittest_msplanner.py Wed Sep 16 14:17:12 2009 +0200 +++ b/server/test/unittest_msplanner.py Wed Sep 16 14:24:31 2009 +0200 @@ -348,7 +348,7 @@ def setUp(self): BaseMSPlannerTC.setUp(self) - self.planner = MSPlanner(self.o.schema, self.o._rqlhelper) + self.planner = MSPlanner(self.o.schema, self.repo.vreg.rqlhelper) _test = test_plan @@ -1989,7 +1989,7 @@ self.setup() self.add_source(FakeCardSource, 'cards') self.add_source(FakeCardSource, 'cards2') - self.planner = MSPlanner(self.o.schema, self.o._rqlhelper) + self.planner = MSPlanner(self.o.schema, self.repo.vreg.rqlhelper) assert repo.sources_by_uri['cards2'].support_relation('multisource_crossed_rel') assert 'multisource_crossed_rel' in repo.sources_by_uri['cards2'].cross_relations assert repo.sources_by_uri['cards'].support_relation('multisource_crossed_rel') @@ -2142,7 +2142,7 @@ def setUp(self): self.setup() self.add_source(FakeVCSSource, 'vcs') - self.planner = MSPlanner(self.o.schema, self.o._rqlhelper) + self.planner = MSPlanner(self.o.schema, self.repo.vreg.rqlhelper) _test = test_plan def test_multisource_inlined_rel_skipped(self): diff -r 1ceac4cd4fb7 -r 8604a15995d1 server/test/unittest_rqlrewrite.py --- a/server/test/unittest_rqlrewrite.py Wed Sep 16 14:17:12 2009 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,193 +0,0 @@ -""" - -:organization: Logilab -:copyright: 2001-2009 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. -:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr -:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses -""" -from logilab.common.testlib import unittest_main, TestCase -from logilab.common.testlib import mock_object - -from rql import parse, nodes, RQLHelper - -from cubicweb import Unauthorized -from cubicweb.server.rqlrewrite import RQLRewriter -from cubicweb.devtools import repotest, TestServerConfiguration - -config = TestServerConfiguration('data') -config.bootstrap_cubes() -schema = config.load_schema() -schema.add_relation_def(mock_object(subject='Card', name='in_state', object='State', cardinality='1*')) - -rqlhelper = RQLHelper(schema, special_relations={'eid': 'uid', - 'has_text': 'fti'}) - -def setup_module(*args): - repotest.do_monkey_patch() - -def teardown_module(*args): - repotest.undo_monkey_patch() - -def eid_func_map(eid): - return {1: 'CWUser', - 2: 'Card'}[eid] - -def rewrite(rqlst, snippets_map, kwargs): - class FakeQuerier: - schema = schema - @staticmethod - def solutions(sqlcursor, mainrqlst, kwargs): - rqlhelper.compute_solutions(rqlst, {'eid': eid_func_map}, kwargs=kwargs) - class _rqlhelper: - @staticmethod - def annotate(rqlst): - rqlhelper.annotate(rqlst) - @staticmethod - def simplify(mainrqlst, needcopy=False): - rqlhelper.simplify(rqlst, needcopy) - rewriter = RQLRewriter(FakeQuerier, mock_object(user=(mock_object(eid=1)))) - for v, snippets in snippets_map.items(): - snippets_map[v] = [mock_object(snippet_rqlst=parse('Any X WHERE '+snippet).children[0], - expression='Any X WHERE '+snippet) - for snippet in snippets] - rqlhelper.compute_solutions(rqlst.children[0], {'eid': eid_func_map}, kwargs=kwargs) - solutions = rqlst.children[0].solutions - rewriter.rewrite(rqlst.children[0], snippets_map.items(), solutions, kwargs) - test_vrefs(rqlst.children[0]) - return rewriter.rewritten - -def test_vrefs(node): - vrefmap = {} - for vref in node.iget_nodes(nodes.VariableRef): - vrefmap.setdefault(vref.name, set()).add(vref) - for var in node.defined_vars.itervalues(): - assert not (var.stinfo['references'] ^ vrefmap[var.name]) - assert (var.stinfo['references']) - -class RQLRewriteTC(TestCase): - """a faire: - - * optimisation: detecter les relations utilisees dans les rqlexpressions qui - sont presentes dans la requete de depart pour les reutiliser si possible - - * "has__permission" ? - """ - - def test_base_var(self): - card_constraint = ('X in_state S, U in_group G, P require_state S,' - 'P name "read", P require_group G') - rqlst = parse('Card C') - rewrite(rqlst, {'C': (card_constraint,)}, {}) - self.failUnlessEqual(rqlst.as_string(), - u"Any C WHERE C is Card, B eid %(D)s, " - "EXISTS(C in_state A, B in_group E, F require_state A, " - "F name 'read', F require_group E, A is State, E is CWGroup, F is CWPermission)") - - def test_multiple_var(self): - card_constraint = ('X in_state S, U in_group G, P require_state S,' - 'P name "read", P require_group G') - affaire_constraints = ('X ref LIKE "PUBLIC%"', 'U in_group G, G name "public"') - kwargs = {'u':2} - rqlst = parse('Any S WHERE S documented_by C, C eid %(u)s') - rewrite(rqlst, {'C': (card_constraint,), 'S': affaire_constraints}, - kwargs) - self.assertTextEquals(rqlst.as_string(), - "Any S WHERE S documented_by C, C eid %(u)s, B eid %(D)s, " - "EXISTS(C in_state A, B in_group E, F require_state A, " - "F name 'read', F require_group E, A is State, E is CWGroup, F is CWPermission), " - "(EXISTS(S ref LIKE 'PUBLIC%')) OR (EXISTS(B in_group G, G name 'public', G is CWGroup)), " - "S is Affaire") - self.failUnless('D' in kwargs) - - def test_or(self): - constraint = '(X identity U) OR (X in_state ST, CL identity U, CL in_state ST, ST name "subscribed")' - rqlst = parse('Any S WHERE S owned_by C, C eid %(u)s, S is in (CWUser, CWGroup)') - rewrite(rqlst, {'C': (constraint,)}, {'u':1}) - self.failUnlessEqual(rqlst.as_string(), - "Any S WHERE S owned_by C, C eid %(u)s, S is IN(CWUser, CWGroup), A eid %(B)s, " - "EXISTS((C identity A) OR (C in_state D, E identity A, " - "E in_state D, D name 'subscribed'), D is State, E is CWUser)") - - def test_simplified_rqlst(self): - card_constraint = ('X in_state S, U in_group G, P require_state S,' - 'P name "read", P require_group G') - rqlst = parse('Any 2') # this is the simplified rql st for Any X WHERE X eid 12 - rewrite(rqlst, {'2': (card_constraint,)}, {}) - self.failUnlessEqual(rqlst.as_string(), - u"Any 2 WHERE B eid %(C)s, " - "EXISTS(2 in_state A, B in_group D, E require_state A, " - "E name 'read', E require_group D, A is State, D is CWGroup, E is CWPermission)") - - def test_optional_var(self): - card_constraint = ('X in_state S, U in_group G, P require_state S,' - 'P name "read", P require_group G') - rqlst = parse('Any A,C WHERE A documented_by C?') - rewrite(rqlst, {'C': (card_constraint,)}, {}) - self.failUnlessEqual(rqlst.as_string(), - "Any A,C WHERE A documented_by C?, A is Affaire " - "WITH C BEING " - "(Any C WHERE C in_state B, D in_group F, G require_state B, G name 'read', " - "G require_group F, D eid %(A)s, C is Card)") - rqlst = parse('Any A,C,T WHERE A documented_by C?, C title T') - rewrite(rqlst, {'C': (card_constraint,)}, {}) - self.failUnlessEqual(rqlst.as_string(), - "Any A,C,T WHERE A documented_by C?, A is Affaire " - "WITH C,T BEING " - "(Any C,T WHERE C in_state B, D in_group F, G require_state B, G name 'read', " - "G require_group F, C title T, D eid %(A)s, C is Card)") - - def test_relation_optimization(self): - # since Card in_state State as monovalued cardinality, the in_state - # relation used in the rql expression can be ignored and S replaced by - # the variable from the incoming query - card_constraint = ('X in_state S, U in_group G, P require_state S,' - 'P name "read", P require_group G') - rqlst = parse('Card C WHERE C in_state STATE') - rewrite(rqlst, {'C': (card_constraint,)}, {}) - self.failUnlessEqual(rqlst.as_string(), - u"Any C WHERE C in_state STATE, C is Card, A eid %(B)s, " - "EXISTS(A in_group D, E require_state STATE, " - "E name 'read', E require_group D, D is CWGroup, E is CWPermission), " - "STATE is State") - - def test_unsupported_constraint_1(self): - # CWUser doesn't have require_permission - trinfo_constraint = ('X wf_info_for Y, Y require_permission P, P name "read"') - rqlst = parse('Any U,T WHERE U is CWUser, T wf_info_for U') - self.assertRaises(Unauthorized, rewrite, rqlst, {'T': (trinfo_constraint,)}, {}) - - def test_unsupported_constraint_2(self): - trinfo_constraint = ('X wf_info_for Y, Y require_permission P, P name "read"') - rqlst = parse('Any U,T WHERE U is CWUser, T wf_info_for U') - rewrite(rqlst, {'T': (trinfo_constraint, 'X wf_info_for Y, Y in_group G, G name "managers"')}, {}) - self.failUnlessEqual(rqlst.as_string(), - u"Any U,T WHERE U is CWUser, T wf_info_for U, " - "EXISTS(U in_group B, B name 'managers', B is CWGroup), T is TrInfo") - - def test_unsupported_constraint_3(self): - self.skip('raise unauthorized for now') - trinfo_constraint = ('X wf_info_for Y, Y require_permission P, P name "read"') - rqlst = parse('Any T WHERE T wf_info_for X') - rewrite(rqlst, {'T': (trinfo_constraint, 'X in_group G, G name "managers"')}, {}) - self.failUnlessEqual(rqlst.as_string(), - u'XXX dunno what should be generated') - - def test_add_ambiguity_exists(self): - constraint = ('X concerne Y') - rqlst = parse('Affaire X') - rewrite(rqlst, {'X': (constraint,)}, {}) - self.failUnlessEqual(rqlst.as_string(), - u"Any X WHERE X is Affaire, (((EXISTS(X concerne A, A is Division)) OR (EXISTS(X concerne D, D is SubDivision))) OR (EXISTS(X concerne C, C is Societe))) OR (EXISTS(X concerne B, B is Note))") - - def test_add_ambiguity_outerjoin(self): - constraint = ('X concerne Y') - rqlst = parse('Any X,C WHERE X? documented_by C') - rewrite(rqlst, {'X': (constraint,)}, {}) - # ambiguity are kept in the sub-query, no need to be resolved using OR - self.failUnlessEqual(rqlst.as_string(), - u"Any X,C WHERE X? documented_by C, C is Card WITH X BEING (Any X WHERE X concerne A, X is Affaire)") - - - -if __name__ == '__main__': - unittest_main() diff -r 1ceac4cd4fb7 -r 8604a15995d1 test/data/rewrite/__init__.py diff -r 1ceac4cd4fb7 -r 8604a15995d1 test/data/rewrite/bootstrap_cubes --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/data/rewrite/bootstrap_cubes Wed Sep 16 14:24:31 2009 +0200 @@ -0,0 +1,1 @@ +card, person diff -r 1ceac4cd4fb7 -r 8604a15995d1 test/data/rewrite/schema.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/data/rewrite/schema.py Wed Sep 16 14:24:31 2009 +0200 @@ -0,0 +1,42 @@ +from yams.buildobjs import EntityType, RelationDefinition, String, SubjectRelation +from cubicweb.schema import ERQLExpression + +class Affaire(EntityType): + permissions = { + 'read': ('managers', + ERQLExpression('X owned_by U'), ERQLExpression('X concerne S?, S owned_by U')), + 'add': ('managers', ERQLExpression('X concerne S, S owned_by U')), + 'update': ('managers', 'owners', ERQLExpression('X in_state S, S name in ("pitetre", "en cours")')), + 'delete': ('managers', 'owners', ERQLExpression('X concerne S, S owned_by U')), + } + ref = String(fulltextindexed=True, indexed=True, + constraints=[SizeConstraint(16)]) + documented_by = SubjectRelation('Card') + concerne = SubjectRelation(('Societe', 'Note')) + + +class Societe(EntityType): + permissions = { + 'read': ('managers', 'users', 'guests'), + 'update': ('managers', 'owners', ERQLExpression('U login L, X nom L')), + 'delete': ('managers', 'owners', ERQLExpression('U login L, X nom L')), + 'add': ('managers', 'users',) + } + + +class Division(Societe): + __specializes_schema__ = True + + +class Note(EntityType): + pass + + +class require_permission(RelationDefinition): + subject = ('Card', 'Note', 'Person') + object = 'CWPermission' + + +class require_state(RelationDefinition): + subject = 'CWPermission' + object = 'State' diff -r 1ceac4cd4fb7 -r 8604a15995d1 test/unittest_rqlrewrite.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/unittest_rqlrewrite.py Wed Sep 16 14:24:31 2009 +0200 @@ -0,0 +1,193 @@ +""" + +:organization: Logilab +:copyright: 2001-2009 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. +:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr +:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses +""" +from logilab.common.testlib import unittest_main, TestCase +from logilab.common.testlib import mock_object + +from rql import parse, nodes, RQLHelper + +from cubicweb import Unauthorized +from cubicweb.rqlrewrite import RQLRewriter +from cubicweb.devtools import repotest, TestServerConfiguration + +config = TestServerConfiguration('data/rewrite') +config.bootstrap_cubes() +schema = config.load_schema() +schema.add_relation_def(mock_object(subject='Card', name='in_state', object='State', cardinality='1*')) + +rqlhelper = RQLHelper(schema, special_relations={'eid': 'uid', + 'has_text': 'fti'}) + +def setup_module(*args): + repotest.do_monkey_patch() + +def teardown_module(*args): + repotest.undo_monkey_patch() + +def eid_func_map(eid): + return {1: 'CWUser', + 2: 'Card'}[eid] + +def rewrite(rqlst, snippets_map, kwargs): + class FakeVReg: + schema = schema + @staticmethod + def solutions(sqlcursor, mainrqlst, kwargs): + rqlhelper.compute_solutions(rqlst, {'eid': eid_func_map}, kwargs=kwargs) + class rqlhelper: + @staticmethod + def annotate(rqlst): + rqlhelper.annotate(rqlst) + @staticmethod + def simplify(mainrqlst, needcopy=False): + rqlhelper.simplify(rqlst, needcopy) + rewriter = RQLRewriter(mock_object(vreg=FakeVReg, user=(mock_object(eid=1)))) + for v, snippets in snippets_map.items(): + snippets_map[v] = [mock_object(snippet_rqlst=parse('Any X WHERE '+snippet).children[0], + expression='Any X WHERE '+snippet) + for snippet in snippets] + rqlhelper.compute_solutions(rqlst.children[0], {'eid': eid_func_map}, kwargs=kwargs) + solutions = rqlst.children[0].solutions + rewriter.rewrite(rqlst.children[0], snippets_map.items(), solutions, kwargs) + test_vrefs(rqlst.children[0]) + return rewriter.rewritten + +def test_vrefs(node): + vrefmap = {} + for vref in node.iget_nodes(nodes.VariableRef): + vrefmap.setdefault(vref.name, set()).add(vref) + for var in node.defined_vars.itervalues(): + assert not (var.stinfo['references'] ^ vrefmap[var.name]) + assert (var.stinfo['references']) + +class RQLRewriteTC(TestCase): + """a faire: + + * optimisation: detecter les relations utilisees dans les rqlexpressions qui + sont presentes dans la requete de depart pour les reutiliser si possible + + * "has__permission" ? + """ + + def test_base_var(self): + card_constraint = ('X in_state S, U in_group G, P require_state S,' + 'P name "read", P require_group G') + rqlst = parse('Card C') + rewrite(rqlst, {('C', 'X'): (card_constraint,)}, {}) + self.failUnlessEqual(rqlst.as_string(), + u"Any C WHERE C is Card, B eid %(D)s, " + "EXISTS(C in_state A, B in_group E, F require_state A, " + "F name 'read', F require_group E, A is State, E is CWGroup, F is CWPermission)") + + def test_multiple_var(self): + card_constraint = ('X in_state S, U in_group G, P require_state S,' + 'P name "read", P require_group G') + affaire_constraints = ('X ref LIKE "PUBLIC%"', 'U in_group G, G name "public"') + kwargs = {'u':2} + rqlst = parse('Any S WHERE S documented_by C, C eid %(u)s') + rewrite(rqlst, {('C', 'X'): (card_constraint,), ('S', 'X'): affaire_constraints}, + kwargs) + self.assertTextEquals(rqlst.as_string(), + "Any S WHERE S documented_by C, C eid %(u)s, B eid %(D)s, " + "EXISTS(C in_state A, B in_group E, F require_state A, " + "F name 'read', F require_group E, A is State, E is CWGroup, F is CWPermission), " + "(EXISTS(S ref LIKE 'PUBLIC%')) OR (EXISTS(B in_group G, G name 'public', G is CWGroup)), " + "S is Affaire") + self.failUnless('D' in kwargs) + + def test_or(self): + constraint = '(X identity U) OR (X in_state ST, CL identity U, CL in_state ST, ST name "subscribed")' + rqlst = parse('Any S WHERE S owned_by C, C eid %(u)s, S is in (CWUser, CWGroup)') + rewrite(rqlst, {('C', 'X'): (constraint,)}, {'u':1}) + self.failUnlessEqual(rqlst.as_string(), + "Any S WHERE S owned_by C, C eid %(u)s, S is IN(CWUser, CWGroup), A eid %(B)s, " + "EXISTS((C identity A) OR (C in_state D, E identity A, " + "E in_state D, D name 'subscribed'), D is State, E is CWUser)") + + def test_simplified_rqlst(self): + card_constraint = ('X in_state S, U in_group G, P require_state S,' + 'P name "read", P require_group G') + rqlst = parse('Any 2') # this is the simplified rql st for Any X WHERE X eid 12 + rewrite(rqlst, {('2', 'X'): (card_constraint,)}, {}) + self.failUnlessEqual(rqlst.as_string(), + u"Any 2 WHERE B eid %(C)s, " + "EXISTS(2 in_state A, B in_group D, E require_state A, " + "E name 'read', E require_group D, A is State, D is CWGroup, E is CWPermission)") + + def test_optional_var(self): + card_constraint = ('X in_state S, U in_group G, P require_state S,' + 'P name "read", P require_group G') + rqlst = parse('Any A,C WHERE A documented_by C?') + rewrite(rqlst, {('C', 'X'): (card_constraint,)}, {}) + self.failUnlessEqual(rqlst.as_string(), + "Any A,C WHERE A documented_by C?, A is Affaire " + "WITH C BEING " + "(Any C WHERE C in_state B, D in_group F, G require_state B, G name 'read', " + "G require_group F, D eid %(A)s, C is Card)") + rqlst = parse('Any A,C,T WHERE A documented_by C?, C title T') + rewrite(rqlst, {('C', 'X'): (card_constraint,)}, {}) + self.failUnlessEqual(rqlst.as_string(), + "Any A,C,T WHERE A documented_by C?, A is Affaire " + "WITH C,T BEING " + "(Any C,T WHERE C in_state B, D in_group F, G require_state B, G name 'read', " + "G require_group F, C title T, D eid %(A)s, C is Card)") + + def test_relation_optimization(self): + # since Card in_state State as monovalued cardinality, the in_state + # relation used in the rql expression can be ignored and S replaced by + # the variable from the incoming query + card_constraint = ('X in_state S, U in_group G, P require_state S,' + 'P name "read", P require_group G') + rqlst = parse('Card C WHERE C in_state STATE') + rewrite(rqlst, {('C', 'X'): (card_constraint,)}, {}) + self.failUnlessEqual(rqlst.as_string(), + u"Any C WHERE C in_state STATE, C is Card, A eid %(B)s, " + "EXISTS(A in_group D, E require_state STATE, " + "E name 'read', E require_group D, D is CWGroup, E is CWPermission), " + "STATE is State") + + def test_unsupported_constraint_1(self): + # CWUser doesn't have require_permission + trinfo_constraint = ('X wf_info_for Y, Y require_permission P, P name "read"') + rqlst = parse('Any U,T WHERE U is CWUser, T wf_info_for U') + self.assertRaises(Unauthorized, rewrite, rqlst, {('T', 'X'): (trinfo_constraint,)}, {}) + + def test_unsupported_constraint_2(self): + trinfo_constraint = ('X wf_info_for Y, Y require_permission P, P name "read"') + rqlst = parse('Any U,T WHERE U is CWUser, T wf_info_for U') + rewrite(rqlst, {('T', 'X'): (trinfo_constraint, 'X wf_info_for Y, Y in_group G, G name "managers"')}, {}) + self.failUnlessEqual(rqlst.as_string(), + u"Any U,T WHERE U is CWUser, T wf_info_for U, " + "EXISTS(U in_group B, B name 'managers', B is CWGroup), T is TrInfo") + + def test_unsupported_constraint_3(self): + self.skip('raise unauthorized for now') + trinfo_constraint = ('X wf_info_for Y, Y require_permission P, P name "read"') + rqlst = parse('Any T WHERE T wf_info_for X') + rewrite(rqlst, {('T', 'X'): (trinfo_constraint, 'X in_group G, G name "managers"')}, {}) + self.failUnlessEqual(rqlst.as_string(), + u'XXX dunno what should be generated') + + def test_add_ambiguity_exists(self): + constraint = ('X concerne Y') + rqlst = parse('Affaire X') + rewrite(rqlst, {('X', 'X'): (constraint,)}, {}) + self.failUnlessEqual(rqlst.as_string(), + u"Any X WHERE X is Affaire, ((EXISTS(X concerne A, A is Division)) OR (EXISTS(X concerne C, C is Societe))) OR (EXISTS(X concerne B, B is Note))") + + def test_add_ambiguity_outerjoin(self): + constraint = ('X concerne Y') + rqlst = parse('Any X,C WHERE X? documented_by C') + rewrite(rqlst, {('X', 'X'): (constraint,)}, {}) + # ambiguity are kept in the sub-query, no need to be resolved using OR + self.failUnlessEqual(rqlst.as_string(), + u"Any X,C WHERE X? documented_by C, C is Card WITH X BEING (Any X WHERE X concerne A, X is Affaire)") + + + +if __name__ == '__main__': + unittest_main()