# HG changeset patch # User sylvain.thenault@logilab.fr # Date 1242222407 -7200 # Node ID 01245e2a777d192ac3996ab708e62637348a082d # Parent f0fb914e57db59c377dc92fc6cabd5f606f96a14 ms planning fix diff -r f0fb914e57db -r 01245e2a777d server/msplanner.py --- a/server/msplanner.py Wed May 13 15:06:02 2009 +0200 +++ b/server/msplanner.py Wed May 13 15:46:47 2009 +0200 @@ -16,7 +16,7 @@ "cross_relations" set in the source's mapping file and it that case, we'll consider that we can also find in the system source some relation between X and Y coming from different sources. - + * if "relation" isn't supported by the external source but X or Y types (or both) are, we suppose by default that can find in the system source some relation where X and/or Y come from the external source. You @@ -49,7 +49,7 @@ 1. return the result of CWUser X WHERE X in_group G, G name 'users' from system source, that's enough (optimization of the sql querier will avoid join on CWUser, so we will directly get local eids) - + :CWUser X,L WHERE X in_group G, X login L, G name 'users': 1. fetch Any X,L WHERE X is CWUser, X login L from both sources, store concatenation of results into a temporary table @@ -98,7 +98,7 @@ AbstractSource.dont_cross_relations = () AbstractSource.cross_relations = () - + def need_aggr_step(select, sources, stepdefs=None): """return True if a temporary table is necessary to store some partial results to execute the given query @@ -169,7 +169,7 @@ for part in subparts: newnode.append(part) return newnode - + def same_scope(var): """return true if the variable is always used in the same scope""" try: @@ -181,7 +181,7 @@ return False var.stinfo['samescope'] = True return True - + ################################################################################ class PartPlanInformation(object): @@ -198,19 +198,19 @@ the execution plan :attr rqlst: the original rql syntax tree handled by this part - + :attr needsplit: bool telling if the query has to be split into multiple steps for execution or if it can be executed at once - + :attr temptable: a SQL temporary table name or None, if necessary to handle aggregate / sorting for this part of the query - + :attr finaltable: a SQL table name or None, if results for this part of the query should be written into a temporary table (usually shared by multiple PPI) - + :attr sourcesterms: a dictionary {source : {term: set([solution index, ])}} telling for each source which terms are supported for which solutions. A "term" may be @@ -262,23 +262,23 @@ print 'sourcesterms:' for source, terms in self.sourcesterms.items(): print source, terms - + def copy_solutions(self, solindices): return [self._solutions[solidx].copy() for solidx in solindices] - + @property @cached def part_sources(self): if self._sourcesterms: return tuple(sorted(self._sourcesterms)) return (self.system_source,) - + @property @cached def _sys_source_set(self): return frozenset((self.system_source, solindex) - for solindex in self._solindices) - + for solindex in self._solindices) + @cached def _norel_support_set(self, relation): """return a set of (source, solindex) where source doesn't support the @@ -340,7 +340,7 @@ # query if not varobj._q_invariant and any(ifilterfalse( source.support_relation, (r.r_type for r in rels))): - self.needsplit = True + self.needsplit = True # add source for rewritten constants to sourcesterms for vconsts in self.rqlst.stinfo['rewritten'].itervalues(): const = vconsts[0] @@ -397,7 +397,7 @@ self._linkedterms.setdefault(lhsv, set()).add((rhsv, rel)) self._linkedterms.setdefault(rhsv, set()).add((lhsv, rel)) return termssources - + def _handle_cross_relation(self, rel, relsources, termssources): for source in relsources: if rel.r_type in source.cross_relations: @@ -422,7 +422,7 @@ break else: self._sourcesterms.setdefault(source, {})[rel] = set(self._solindices) - + def _remove_invalid_sources(self, termssources): """removes invalid sources from `sourcesterms` member according to traversed relations and their properties (which sources support them, @@ -455,7 +455,7 @@ continue self._remove_term_sources(lhsv, rel, rhsv, termssources) self._remove_term_sources(rhsv, rel, lhsv, termssources) - + def _extern_term(self, term, termssources, inserted): var = term.variable if var.stinfo['constnode']: @@ -471,7 +471,7 @@ if not termv in termssources: termssources[termv] = self._term_sources(termv) return termv - + def _remove_sources_until_stable(self, term, termssources): sourcesterms = self._sourcesterms for oterm, rel in self._linkedterms.get(term, ()): @@ -506,10 +506,10 @@ self._remove_term_sources(term, rel, oterm, termssources) if not need_ancestor_scope or is_ancestor(oterm.scope, term.scope): self._remove_term_sources(oterm, rel, term, termssources) - + def _remove_term_sources(self, term, rel, oterm, termssources): """remove invalid sources for term according to oterm's sources and the - relation between those two terms. + relation between those two terms. """ norelsup = self._norel_support_set(rel) termsources = termssources[term] @@ -528,21 +528,23 @@ self._remove_sources(term, invalid_sources) termsources -= invalid_sources self._remove_sources_until_stable(term, termssources) - + if isinstance(oterm, Constant): + self._remove_sources(oterm, invalid_sources) + def _compute_needsplit(self): """tell according to sourcesterms if the rqlst has to be splitted for execution among multiple sources - + the execution has to be split if * a source support an entity (non invariant) but doesn't support a relation on it * a source support an entity which is accessed by an optional relation - * there is more than one source and either all sources'supported + * there is more than one source and either all sources'supported variable/solutions are not equivalent or multiple variables have to be fetched from some source """ # NOTE: < 2 since may be 0 on queries such as Any X WHERE X eid 2 - if len(self._sourcesterms) < 2: + if len(self._sourcesterms) < 2: self.needsplit = False elif not self.needsplit: if not allequals(self._sourcesterms.itervalues()): @@ -576,7 +578,7 @@ if not r is rel and self._repo.is_multi_sources_relation(r.r_type)): return True return False - + def _set_source_for_term(self, source, term): self._sourcesterms.setdefault(source, {})[term] = set(self._solindices) @@ -603,7 +605,7 @@ try: sourcesterms[source][term].remove(solindex) except KeyError: - return # may occur with subquery column alias + continue # may occur with subquery column alias if not sourcesterms[source][term]: del sourcesterms[source][term] if not sourcesterms[source]: @@ -611,7 +613,7 @@ def crossed_relation(self, source, relation): return relation in self._crossrelations.get(source, ()) - + def part_steps(self): """precompute necessary part steps before generating actual rql for each step. This is necessary to know if an aggregate step will be @@ -758,10 +760,10 @@ # ensure relation is using '=' operator, else we rely on a # sqlgenerator side effect (it won't insert an inequality operator # in this case) - relation.children[1].operator = '=' + relation.children[1].operator = '=' terms.append(newvar) needsel.add(newvar.name) - + def _choose_term(self, sourceterms): """pick one term among terms supported by a source, which will be used as a base to generate an execution step @@ -798,7 +800,7 @@ # whatever (relation) term = iter(sourceterms).next() return term, sourceterms.pop(term) - + def _expand_sources(self, selected_source, term, solindices): """return all sources supporting given term / solindices""" sources = [selected_source] @@ -806,7 +808,7 @@ for source in sourcesterms.keys(): if source is selected_source: continue - if not (term in sourcesterms[source] and + if not (term in sourcesterms[source] and solindices.issubset(sourcesterms[source][term])): continue sources.append(source) @@ -818,7 +820,7 @@ if not sourcesterms[source]: del sourcesterms[source] return sources - + def _expand_terms(self, term, sources, sourceterms, scope, solindices): terms = [term] sources = sorted(sources) @@ -876,7 +878,7 @@ modified = True self._cleanup_sourcesterms(sources, solindices, term) return terms - + def _cleanup_sourcesterms(self, sources, solindices, term=None): """remove solutions so we know they are already processed""" for source in sources: @@ -901,7 +903,7 @@ #assert term in cross_terms if not sourceterms: del self._sourcesterms[source] - + def merge_input_maps(self, allsolindices): """inputmaps is a dictionary with tuple of solution indices as key with an associated input map as value. This function compute for each @@ -911,7 +913,7 @@ inputmaps = {(0, 1, 2): {'A': 't1.login1', 'U': 't1.C0', 'U.login': 't1.login1'}, (1,): {'X': 't2.C0', 'T': 't2.C1'}} return : [([1], {'A': 't1.login1', 'U': 't1.C0', 'U.login': 't1.login1', - 'X': 't2.C0', 'T': 't2.C1'}), + 'X': 't2.C0', 'T': 't2.C1'}), ([0,2], {'A': 't1.login1', 'U': 't1.C0', 'U.login': 't1.login1'})] """ if not self._inputmaps: @@ -980,10 +982,10 @@ decompose the RQL query according to sources'schema """ - + def build_select_plan(self, plan, rqlst): """build execution plan for a SELECT RQL query - + the rqlst should not be tagged at this point """ if server.DEBUG: @@ -1030,7 +1032,7 @@ inputmap[colalias.name] = '%s.C%s' % (temptable, i) ppi.plan.add_step(sstep) return inputmap - + def _union_plan(self, plan, union, ppis, temptable=None): tosplit, cango, allsources = [], {}, set() for planinfo in ppis: @@ -1088,7 +1090,7 @@ return steps # internal methods for multisources decomposition ######################### - + def split_part(self, ppi, temptable): ppi.finaltable = temptable plan = ppi.plan @@ -1172,7 +1174,7 @@ step.set_limit_offset(select.limit, select.offset) return step - + class UnsupportedBranch(Exception): pass @@ -1185,7 +1187,7 @@ self.hasaggrstep = self.ppi.temptable self.extneedsel = frozenset(vref.name for sortterm in ppi.rqlst.orderby for vref in sortterm.iget_nodes(VariableRef)) - + def _rqlst_accept(self, rqlst, node, newroot, terms, setfunc=None): try: newrestr, node_ = node.accept(self, newroot, terms[:]) @@ -1293,7 +1295,7 @@ if server.DEBUG: print '--->', newroot return newroot, self.insertedvars - + def visit_and(self, node, newroot, terms): subparts = [] for i in xrange(len(node.children)): @@ -1330,7 +1332,7 @@ if termsources and termsources != self.sources: return False return True - + def visit_relation(self, node, newroot, terms): if not node.is_types_restriction(): if node in self.skip and self.solindices.issubset(self.skip[node]): @@ -1368,7 +1370,7 @@ if not ored: self.skip.setdefault(node, set()).update(self.solindices) else: - self.mayneedvar.setdefault((node.children[0].name, rschema), []).append( (res, ored) ) + self.mayneedvar.setdefault((node.children[0].name, rschema), []).append( (res, ored) ) else: assert len(vrefs) == 1 vref = vrefs[0] @@ -1391,7 +1393,7 @@ if any(v for v, _ in var.stinfo['attrvars'] if not v in terms): return False return True - + def visit_exists(self, node, newroot, terms): newexists = node.__class__() self.scopes = {node: newexists} @@ -1400,18 +1402,18 @@ return None, node newexists.set_where(subparts[0]) return newexists, node - + def visit_not(self, node, newroot, terms): subparts, node = self._visit_children(node, newroot, terms) if not subparts: return None, node return copy_node(newroot, node, subparts), node - + def visit_group(self, node, newroot, terms): if not self.final: return None, node return self.visit_default(node, newroot, terms) - + def visit_variableref(self, node, newroot, terms): if self.use_only_defined: if not node.variable.name in newroot.defined_vars: @@ -1426,14 +1428,14 @@ def visit_constant(self, node, newroot, terms): return copy_node(newroot, node), node - + def visit_default(self, node, newroot, terms): subparts, node = self._visit_children(node, newroot, terms) return copy_node(newroot, node, subparts), node - + visit_comparison = visit_mathexpression = visit_constant = visit_function = visit_default visit_sort = visit_sortterm = visit_default - + def _visit_children(self, node, newroot, terms): subparts = [] for i in xrange(len(node.children)): @@ -1444,14 +1446,14 @@ if newchild is not None: subparts.append(newchild) return subparts, node - + def process_selection(self, newroot, terms, rqlst): if self.final: for term in rqlst.selection: newroot.append_selected(term.copy(newroot)) for vref in term.get_nodes(VariableRef): self.needsel.add(vref.name) - return + return for term in rqlst.selection: vrefs = term.get_nodes(VariableRef) if vrefs: @@ -1471,7 +1473,7 @@ for vref in supportedvars: if not vref in newroot.get_selected_variables(): newroot.append_selected(VariableRef(newroot.get_variable(vref.name))) - + def add_necessary_selection(self, newroot, terms): selected = tuple(newroot.get_selected_variables()) for varname in terms: diff -r f0fb914e57db -r 01245e2a777d server/test/unittest_msplanner.py --- a/server/test/unittest_msplanner.py Wed May 13 15:06:02 2009 +0200 +++ b/server/test/unittest_msplanner.py Wed May 13 15:46:47 2009 +0200 @@ -24,15 +24,15 @@ def syntax_tree_search(self, *args, **kwargs): return [] - + class FakeCardSource(AbstractSource): uri = 'ccc' support_entities = {'Card': True, 'Note': True, 'State': True} support_relations = {'in_state': True, 'multisource_rel': True, 'multisource_inlined_rel': True, 'multisource_crossed_rel': True} - dont_cross_relations = set(('fiche',)) + dont_cross_relations = set(('fiche', 'in_state')) cross_relations = set(('multisource_crossed_rel',)) - + def syntax_tree_search(self, *args, **kwargs): return [] @@ -53,19 +53,19 @@ clear_cache(repo, 'can_cross_relation') clear_cache(repo, 'is_multi_sources_relation') # XXX source_defs - + # keep cnx so it's not garbage collected and the associated session is closed repo, cnx = init_test_database('sqlite') class BaseMSPlannerTC(BasePlannerTC): """test planner related feature on a 3-sources repository: - + * system source supporting everything * ldap source supporting CWUser * rql source supporting Card """ repo = repo - + def setUp(self): #_QuerierTC.setUp(self) clear_cache(repo, 'rel_type_sources') @@ -84,7 +84,7 @@ self.prevrqlexpr_user = userreadperms[-1] userreadperms[-1] = ERQLExpression('X owned_by U') self.schema['CWUser']._groups['read'] = tuple(userreadperms) - + self.sources = self.o._repo.sources self.system = self.sources[-1] self.sources.append(FakeUserROSource(self.o._repo, self.o.schema, @@ -97,7 +97,7 @@ self.rql = self.sources[-1] do_monkey_patch() clear_ms_caches(repo) - + def tearDown(self): undo_monkey_patch() del self.sources[-1] @@ -107,20 +107,20 @@ # restore hijacked security self.restore_orig_affaire_security() self.restore_orig_euser_security() - + def restore_orig_affaire_security(self): affreadperms = list(self.schema['Affaire']._groups['read']) affreadperms[-1] = self.prevrqlexpr_affaire self.schema['Affaire']._groups['read'] = tuple(affreadperms) clear_cache(self.schema['Affaire'], 'ERSchema_get_rqlexprs') - + def restore_orig_euser_security(self): userreadperms = list(self.schema['CWUser']._groups['read']) userreadperms[-1] = self.prevrqlexpr_user self.schema['CWUser']._groups['read'] = tuple(userreadperms) clear_cache(self.schema['CWUser'], 'ERSchema_get_rqlexprs') - + class PartPlanInformationTC(BaseMSPlannerTC): def _test(self, rql, *args): @@ -140,44 +140,44 @@ self.assertEquals(ppi._sourcesterms, sourcesterms) self.assertEquals(ppi.needsplit, needsplit) - + def test_simple_system_only(self): """retrieve entities only supported by the system source""" self._test('CWGroup X', {self.system: {'X': s[0]}}, False) - + def test_simple_system_ldap(self): """retrieve CWUser X from both sources and return concatenation of results """ self._test('CWUser X', {self.system: {'X': s[0]}, self.ldap: {'X': s[0]}}, False) - + def test_simple_system_rql(self): """retrieve Card X from both sources and return concatenation of results """ self._test('Any X, XT WHERE X is Card, X title XT', {self.system: {'X': s[0]}, self.rql: {'X': s[0]}}, False) - + def test_simple_eid_specified(self): """retrieve CWUser X from system source (eid is specified, can locate the entity) """ ueid = self.session.user.eid self._test('Any X,L WHERE X eid %(x)s, X login L', {'x': ueid}, {self.system: {'X': s[0]}}, False) - + def test_simple_eid_invariant(self): """retrieve CWUser X from system source (eid is specified, can locate the entity) """ ueid = self.session.user.eid self._test('Any X WHERE X eid %(x)s', {'x': ueid}, {self.system: {'x': s[0]}}, False) - + def test_simple_invariant(self): """retrieve CWUser X from system source only (X is invariant and in_group not supported by ldap source) """ self._test('Any X WHERE X is CWUser, X in_group G, G name "users"', {self.system: {'X': s[0], 'G': s[0], 'in_group': s[0]}}, False) - + def test_security_has_text(self): """retrieve CWUser X from system source only (has_text not supported by ldap source) """ @@ -185,7 +185,7 @@ # with ambigous query (eg only considering the first solution) self._test('CWUser X WHERE X has_text "bla"', {self.system: {'X': s[0]}}, False) - + def test_complex_base(self): """ 1. retrieve Any X, L WHERE X is CWUser, X login L from system and ldap sources, store @@ -202,7 +202,7 @@ 1. retrieve Any X,AA WHERE X modification_date AA from system and ldap sources, store concatenation of results into a temporary table 2. return the result of Any X,AA ORDERBY AA WHERE %s owned_by X, X modification_date AA - on the system source + on the system source """ ueid = self.session.user.eid self._test('Any X,AA ORDERBY AA WHERE E eid %(x)s, E owned_by X, X modification_date AA', {'x': ueid}, @@ -214,7 +214,7 @@ 1. retrieve Any X,L,AA WHERE X login L, X modification_date AA from system and ldap sources, store concatenation of results into a temporary table 2. return the result of Any X,L,AA WHERE %s owned_by X, X login L, X modification_date AA - on the system source + on the system source """ ueid = self.session.user.eid self._test('Any X,L,AA WHERE E eid %(x)s, E owned_by X, X login L, X modification_date AA', {'x': ueid}, @@ -233,18 +233,18 @@ 1. retrieve Any X,A,Y,B WHERE X login A, Y login B from system and ldap sources, store cartesian product of results into a temporary table 2. return the result of Any X,Y WHERE X login 'syt', Y login 'adim' - on the system source + on the system source """ ueid = self.session.user.eid self._test('Any X,Y WHERE X login "syt", Y login "adim"', {'x': ueid}, {self.system: {'Y': s[0], 'X': s[0]}, self.ldap: {'Y': s[0], 'X': s[0]}}, True) - + def test_complex_aggregat(self): solindexes = set(range(len([e for e in self.schema.entities() if not e.is_final()]))) self._test('Any MAX(X)', {self.system: {'X': solindexes}}, False) - + def test_complex_optional(self): ueid = self.session.user.eid self._test('Any U WHERE WF wf_info_for X, X eid %(x)s, WF owned_by U?, WF from_state FS', {'x': ueid}, @@ -252,7 +252,7 @@ 'from_state': s[0], 'owned_by': s[0], 'wf_info_for': s[0], 'x': s[0]}}, False) - + def test_exists4(self): """ State S could come from both rql source and system source, @@ -262,7 +262,7 @@ self._test('Any G,L WHERE X in_group G, X login L, G name "managers", ' 'EXISTS(X copain T, T login L, T login in ("comme", "cochon")) OR ' 'EXISTS(X in_state S, S name "pascontent", NOT X copain T2, T2 login "billy")', - {self.system: {'X': s[0], 'S': s[0], 'T2': s[0], 'T': s[0], 'G': s[0], 'copain': s[0], 'in_group': s[0]}, + {self.system: {'X': s[0], 'S': s[0], 'T2': s[0], 'T': s[0], 'G': s[0], 'copain': s[0], 'in_group': s[0]}, self.ldap: {'X': s[0], 'T2': s[0], 'T': s[0]}}, True) @@ -271,18 +271,18 @@ {self.system: {'X': s[0, 1, 2], 'S': s[0, 1, 2]}, self.rql: {'X': s[2], 'S': s[2]}}, True) - + def test_not_relation_need_split(self): self._test('Any SN WHERE NOT X in_state S, S name SN', {self.rql: {'X': s[2], 'S': s[0, 1, 2]}, self.system: {'X': s[0, 1, 2], 'S': s[0, 1, 2]}}, True) - + def test_not_relation_no_split_external(self): repo._type_source_cache[999999] = ('Note', 'cards', 999999) # similar to the above test but with an eid coming from the external source. # the same plan may be used, since we won't find any record in the system source - # linking 9999999 to a state + # linking 9999999 to a state self._test('Any SN WHERE NOT X in_state S, X eid %(x)s, S name SN', {'x': 999999}, {self.rql: {'x': s[0], 'S': s[0]}, @@ -303,7 +303,7 @@ 'require_permission': s[0], 'in_group': s[0], 'P': s[0], 'require_group': s[0], 'u': s[0]}}, False) - + def test_delete_relation1(self): ueid = self.session.user.eid self._test('Any X, Y WHERE X created_by Y, X eid %(x)s, NOT Y eid %(y)s', @@ -318,7 +318,7 @@ {'x': 999999,}, {self.rql: {'Y': s[0]}, self.system: {'Y': s[0], 'x': s[0]}}, True) - + def test_crossed_relation_eid_1_invariant(self): repo._type_source_cache[999999] = ('Note', 'system', 999999) self._test('Any Y WHERE X eid %(x)s, X multisource_crossed_rel Y', @@ -341,7 +341,7 @@ {self.rql: {'X': s[0], 'AD': s[0], 'multisource_crossed_rel': s[0], 'x': s[0]}, self.system: {'X': s[0], 'AD': s[0], 'multisource_crossed_rel': s[0], 'x': s[0]}}, True) - + def test_version_crossed_depends_on_2(self): repo._type_source_cache[999999] = ('Note', 'system', 999999) self._test('Any X,AD,AE WHERE E eid %(x)s, E multisource_crossed_rel X, X in_state AD, AD name AE', @@ -356,11 +356,11 @@ self._test('Any S,T WHERE S eid %(s)s, N eid %(n)s, N type T, N is Note, S is State', {'n': 999999, 's': 999998}, {self.rql: {'s': s[0], 'N': s[0]}}, False) - + + - class MSPlannerTC(BaseMSPlannerTC): - + def setUp(self): BaseMSPlannerTC.setUp(self) self.planner = MSPlanner(self.o.schema, self.o._rqlhelper) @@ -387,14 +387,14 @@ self._test('CWGroup X LIMIT 10 OFFSET 10', [('OneFetchStep', [('Any X LIMIT 10 OFFSET 10 WHERE X is CWGroup', [{'X': 'CWGroup'}])], 10, 10, [self.system], {}, [])]) - + def test_simple_system_ldap(self): """retrieve CWUser X from both sources and return concatenation of results """ self._test('CWUser X', [('OneFetchStep', [('Any X WHERE X is CWUser', [{'X': 'CWUser'}])], None, None, [self.ldap, self.system], {}, [])]) - + def test_simple_system_ldap_limit(self): """retrieve CWUser X from both sources and return concatenation of results """ @@ -428,14 +428,14 @@ [self.ldap, self.system], {}, {'COUNT(X)': 'table0.C0'}, []), ]), ]) - + def test_simple_system_rql(self): """retrieve Card X from both sources and return concatenation of results """ self._test('Any X, XT WHERE X is Card, X title XT', [('OneFetchStep', [('Any X,XT WHERE X is Card, X title XT', [{'X': 'Card', 'XT': 'String'}])], None, None, [self.rql, self.system], {}, [])]) - + def test_simple_eid_specified(self): """retrieve CWUser X from system source (eid is specified, can locate the entity) """ @@ -444,7 +444,7 @@ [('OneFetchStep', [('Any X,L WHERE X eid %s, X login L'%ueid, [{'X': 'CWUser', 'L': 'String'}])], None, None, [self.system], {}, [])], {'x': ueid}) - + def test_simple_eid_invariant(self): """retrieve CWUser X from system source (eid is specified, can locate the entity) """ @@ -453,7 +453,7 @@ [('OneFetchStep', [('Any %s'%ueid, [{}])], None, None, [self.system], {}, [])], {'x': ueid}) - + def test_simple_invariant(self): """retrieve CWUser X from system source only (X is invariant and in_group not supported by ldap source) """ @@ -461,7 +461,7 @@ [('OneFetchStep', [('Any X WHERE X is CWUser, X in_group G, G name "users"', [{'X': 'CWUser', 'G': 'CWGroup'}])], None, None, [self.system], {}, [])]) - + def test_complex_base(self): """ 1. retrieve Any X, L WHERE X is CWUser, X login L from system and ldap sources, store @@ -498,7 +498,7 @@ def test_complex_ordered(self): self._test('Any L ORDERBY L WHERE X login L', - [('AggrStep', 'Any L ORDERBY L', None, None, 'table0', None, + [('AggrStep', 'Any L ORDERBY L', None, None, 'table0', None, [('FetchStep', [('Any L WHERE X login L, X is CWUser', [{'X': 'CWUser', 'L': 'String'}])], [self.ldap, self.system], {}, {'X.login': 'table0.C0', 'L': 'table0.C0'}, []), @@ -507,13 +507,13 @@ def test_complex_ordered_limit_offset(self): self._test('Any L ORDERBY L LIMIT 10 OFFSET 10 WHERE X login L', - [('AggrStep', 'Any L ORDERBY L', 10, 10, 'table0', None, + [('AggrStep', 'Any L ORDERBY L', 10, 10, 'table0', None, [('FetchStep', [('Any L WHERE X login L, X is CWUser', [{'X': 'CWUser', 'L': 'String'}])], [self.ldap, self.system], {}, {'X.login': 'table0.C0', 'L': 'table0.C0'}, []), ]) ]) - + def test_complex_invariant_ordered(self): """ 1. retrieve Any X,AA WHERE X modification_date AA from system and ldap sources, store @@ -543,7 +543,7 @@ 1. retrieve Any X,L,AA WHERE X login L, X modification_date AA from system and ldap sources, store concatenation of results into a temporary table 2. return the result of Any X,L,AA WHERE %s owned_by X, X login L, X modification_date AA - on the system source + on the system source """ ueid = self.session.user.eid self._test('Any X,L,AA WHERE E eid %(x)s, E owned_by X, X login L, X modification_date AA', @@ -593,7 +593,7 @@ 2. return content of the table sorted """ self._test('Any X,F ORDERBY F WHERE X firstname F', - [('AggrStep', 'Any X,F ORDERBY F', None, None, 'table0', None, + [('AggrStep', 'Any X,F ORDERBY F', None, None, 'table0', None, [('FetchStep', [('Any X,F WHERE X firstname F, X is CWUser', [{'X': 'CWUser', 'F': 'String'}])], [self.ldap, self.system], {}, @@ -604,13 +604,13 @@ {'X': 'table0.C0', 'X.firstname': 'table0.C1', 'F': 'table0.C1'}, []), ]), ]) - + def test_complex_multiple(self): """ 1. retrieve Any X,A,Y,B WHERE X login A, Y login B from system and ldap sources, store cartesian product of results into a temporary table 2. return the result of Any X,Y WHERE X login 'syt', Y login 'adim' - on the system source + on the system source """ ueid = self.session.user.eid self._test('Any X,Y WHERE X login "syt", Y login "adim"', @@ -627,13 +627,13 @@ None, None, [self.system], {'X': 'table0.C0', 'Y': 'table1.C0'}, []) ], {'x': ueid}) - + def test_complex_multiple_limit_offset(self): """ 1. retrieve Any X,A,Y,B WHERE X login A, Y login B from system and ldap sources, store cartesian product of results into a temporary table 2. return the result of Any X,Y WHERE X login 'syt', Y login 'adim' - on the system source + on the system source """ ueid = self.session.user.eid self._test('Any X,Y LIMIT 10 OFFSET 10 WHERE X login "syt", Y login "adim"', @@ -648,14 +648,14 @@ 10, 10, [self.system], {'X': 'table0.C0', 'Y': 'table1.C0'}, []) ], {'x': ueid}) - + def test_complex_aggregat(self): self._test('Any MAX(X)', [('OneFetchStep', [('Any MAX(X)', X_ALL_SOLS)], None, None, [self.system], {}, []) ]) - + def test_complex_typed_aggregat(self): self._test('Any MAX(X) WHERE X is Card', [('AggrStep', 'Any MAX(X)', None, None, 'table0', None, @@ -664,21 +664,21 @@ [self.rql, self.system], {}, {'MAX(X)': 'table0.C0'}, []) ]) ]) - + def test_complex_greater_eid(self): self._test('Any X WHERE X eid > 12', [('OneFetchStep', [('Any X WHERE X eid > 12', X_ALL_SOLS)], None, None, [self.system], {}, []) ]) - + def test_complex_greater_typed_eid(self): self._test('Any X WHERE X eid > 12, X is Card', [('OneFetchStep', [('Any X WHERE X eid > 12, X is Card', [{'X': 'Card'}])], None, None, [self.system], {}, []) ]) - + def test_complex_optional(self): ueid = self.session.user.eid self._test('Any U WHERE WF wf_info_for X, X eid %(x)s, WF owned_by U?, WF from_state FS', @@ -695,7 +695,7 @@ None, None, [self.system], {}, [])], {'x': ueid}) - + def test_3sources_ambigous(self): self._test('Any X,T WHERE X owned_by U, U login "syt", X title T', [('FetchStep', [('Any X,T WHERE X title T, X is Card', [{'X': 'Card', 'T': 'String'}])], @@ -736,7 +736,7 @@ ]) def test_outer_supported_rel1(self): - # both system and rql support all variables, can be + # both system and rql support all variables, can be self._test('Any X, R WHERE X is Note, X in_state S, X type R, ' 'NOT EXISTS(Y is Note, Y in_state S, Y type R, X identity Y)', [('OneFetchStep', [('Any X,R WHERE X is Note, X in_state S, X type R, NOT EXISTS(Y is Note, Y in_state S, Y type R, X identity Y), S is State', @@ -746,7 +746,7 @@ ]) def test_not_identity(self): - # both system and rql support all variables, can be + # both system and rql support all variables, can be self._test('Any X WHERE NOT X identity U, U eid %s' % self.session.user.eid, [('OneFetchStep', [('Any X WHERE NOT X identity 5, X is CWUser', [{'X': 'CWUser'}])], @@ -769,7 +769,7 @@ None, None, [self.system], {'A': 'table0.C0', 'X': 'table1.C0', 'X.login': 'table1.C1', 'R': 'table1.C1', 'Y.type': 'table0.C1'}, []) ]) - + def test_security_has_text(self): # use a guest user self.session = self._user_session()[1] @@ -795,7 +795,7 @@ None, None, [self.system], {}, []), ]) ]) - + def test_security_has_text_limit_offset(self): # use a guest user self.session = self._user_session()[1] @@ -828,9 +828,9 @@ {'X': 'Folder'}, {'X': 'Image'}, {'X': 'Note'}, {'X': 'Personne'}, {'X': 'Societe'}, {'X': 'State'}, {'X': 'SubDivision'}, {'X': 'Tag'}, {'X': 'Transition'}])], - 10, 10, [self.system], {'X': 'table0.C0'}, []) + 10, 10, [self.system], {'X': 'table0.C0'}, []) ]) - + def test_security_user(self): """a guest user trying to see another user: EXISTS(X owned_by U) is automatically inserted""" # use a guest user @@ -842,7 +842,7 @@ ('OneFetchStep', [('Any X WHERE EXISTS(X owned_by 5), X is CWUser', [{'X': 'CWUser'}])], None, None, [self.system], {'X': 'table0.C0'}, [])]) - + def test_security_complex_has_text(self): # use a guest user self.session = self._user_session()[1] @@ -879,18 +879,18 @@ self.session = self._user_session()[1] self._test('Any MAX(X)', [('FetchStep', [('Any E WHERE E type "X", E is Note', [{'E': 'Note'}])], - [self.rql, self.system], None, {'E': 'table1.C0'}, []), + [self.rql, self.system], None, {'E': 'table1.C0'}, []), ('FetchStep', [('Any X WHERE X is CWUser', [{'X': 'CWUser'}])], [self.ldap, self.system], None, {'X': 'table2.C0'}, []), ('UnionFetchStep', [ ('FetchStep', [('Any X WHERE EXISTS(X owned_by 5), X is Basket', [{'X': 'Basket'}])], - [self.system], {}, {'X': 'table0.C0'}, []), + [self.system], {}, {'X': 'table0.C0'}, []), ('UnionFetchStep', [('FetchStep', [('Any X WHERE X is IN(Card, Note, State)', [{'X': 'Card'}, {'X': 'Note'}, {'X': 'State'}])], [self.rql, self.system], {}, {'X': 'table0.C0'}, []), ('FetchStep', - [('Any X WHERE X is IN(Bookmark, Comment, Division, CWCache, CWConstraint, CWConstraintType, CWEType, CWAttribute, CWGroup, CWRelation, CWPermission, CWProperty, CWRType, Email, EmailAddress, EmailPart, EmailThread, File, Folder, Image, Personne, RQLExpression, Societe, SubDivision, Tag, TrInfo, Transition)', + [('Any X WHERE X is IN(Bookmark, CWAttribute, CWCache, CWConstraint, CWConstraintType, CWEType, CWGroup, CWPermission, CWProperty, CWRType, CWRelation, Comment, Division, Email, EmailAddress, EmailPart, EmailThread, File, Folder, Image, Personne, RQLExpression, Societe, SubDivision, Tag, TrInfo, Transition)', sorted([{'X': 'Bookmark'}, {'X': 'Comment'}, {'X': 'Division'}, {'X': 'CWCache'}, {'X': 'CWConstraint'}, {'X': 'CWConstraintType'}, {'X': 'CWEType'}, {'X': 'CWAttribute'}, {'X': 'CWGroup'}, @@ -906,16 +906,16 @@ [self.system], {'X': 'table2.C0'}, {'X': 'table0.C0'}, []), ('FetchStep', [('Any X WHERE (EXISTS(X owned_by 5)) OR ((((EXISTS(D concerne C?, C owned_by 5, C type "X", X identity D, C is Division, D is Affaire)) OR (EXISTS(H concerne G?, G owned_by 5, G type "X", X identity H, G is SubDivision, H is Affaire))) OR (EXISTS(I concerne F?, F owned_by 5, F type "X", X identity I, F is Societe, I is Affaire))) OR (EXISTS(J concerne E?, E owned_by 5, X identity J, E is Note, J is Affaire))), X is Affaire', [{'C': 'Division', 'E': 'Note', 'D': 'Affaire', 'G': 'SubDivision', 'F': 'Societe', 'I': 'Affaire', 'H': 'Affaire', 'J': 'Affaire', 'X': 'Affaire'}])], - [self.system], {'E': 'table1.C0'}, {'X': 'table0.C0'}, []), + [self.system], {'E': 'table1.C0'}, {'X': 'table0.C0'}, []), ]), ('OneFetchStep', [('Any MAX(X)', X_ALL_SOLS)], None, None, [self.system], {'X': 'table0.C0'}, []) ]) - + def test_security_complex_aggregat2(self): # use a guest user self.session = self._user_session()[1] - self._test('Any ET, COUNT(X) GROUPBY ET ORDERBY ET WHERE X is ET', + self._test('Any ET, COUNT(X) GROUPBY ET ORDERBY ET WHERE X is ET', [('FetchStep', [('Any X WHERE X is IN(Card, Note, State)', [{'X': 'Card'}, {'X': 'Note'}, {'X': 'State'}])], [self.rql, self.system], None, {'X': 'table1.C0'}, []), @@ -939,7 +939,7 @@ [self.system], {'X': 'table3.C0'}, {'ET': 'table0.C0', 'X': 'table0.C1'}, []), # extra UnionFetchStep could be avoided but has no cost, so don't care ('UnionFetchStep', - [('FetchStep', [('Any ET,X WHERE X is ET, ET is CWEType, X is IN(Bookmark, Comment, Division, CWCache, CWConstraint, CWConstraintType, CWEType, CWAttribute, CWGroup, CWRelation, CWPermission, CWProperty, CWRType, Email, EmailAddress, EmailPart, EmailThread, File, Folder, Image, Personne, RQLExpression, Societe, SubDivision, Tag, TrInfo, Transition)', + [('FetchStep', [('Any ET,X WHERE X is ET, ET is CWEType, X is IN(Bookmark, CWAttribute, CWCache, CWConstraint, CWConstraintType, CWEType, CWGroup, CWPermission, CWProperty, CWRType, CWRelation, Comment, Division, Email, EmailAddress, EmailPart, EmailThread, File, Folder, Image, Personne, RQLExpression, Societe, SubDivision, Tag, TrInfo, Transition)', [{'X': 'Bookmark', 'ET': 'CWEType'}, {'X': 'Comment', 'ET': 'CWEType'}, {'X': 'Division', 'ET': 'CWEType'}, {'X': 'CWCache', 'ET': 'CWEType'}, {'X': 'CWConstraint', 'ET': 'CWEType'}, {'X': 'CWConstraintType', 'ET': 'CWEType'}, @@ -1054,7 +1054,7 @@ 10, 10, [self.system], {'X': 'table0.C0', 'X.title': 'table0.C1', 'XT': 'table0.C1', 'U': 'table1.C0'}, []) ]) - + def test_exists_base(self): self._test('Any X,L,S WHERE X in_state S, X login L, EXISTS(X in_group G, G name "bougloup")', [('FetchStep', [('Any X,L WHERE X login L, X is CWUser', [{'X': 'CWUser', 'L': 'String'}])], @@ -1128,7 +1128,7 @@ None, None, [self.system], {'T': 'table0.C0', 'T2': 'table1.C0', 'X': 'table2.C1', 'X.login': 'table2.C0', 'L': 'table2.C0'}, [])]) - + def test_exists_security_no_invariant(self): ueid = self.session.user.eid self._test('Any X,AA,AB,AC,AD ORDERBY AA WHERE X is CWUser, X login AA, X firstname AB, X surname AC, X modification_date AD, A eid %(B)s, \ @@ -1159,10 +1159,10 @@ self._test('Any X, S WHERE X in_state S', [('UnionStep', None, None, [ ('OneFetchStep', [('Any X,S WHERE X in_state S, S is State, X is IN(Affaire, CWUser)', - [{'X': 'Affaire', 'S': 'State'}, {'X': 'CWUser', 'S': 'State'}])], + [{'X': 'Affaire', 'S': 'State'}, {'X': 'CWUser', 'S': 'State'}])], None, None, [self.system], {}, []), ('OneFetchStep', [('Any X,S WHERE X in_state S, S is State, X is Note', - [{'X': 'Note', 'S': 'State'}])], + [{'X': 'Note', 'S': 'State'}])], None, None, [self.rql, self.system], {}, []), ])]) @@ -1223,7 +1223,7 @@ # generation for the external source self._test('Any SN WHERE NOT X in_state S, X eid %(x)s, S name SN', [('OneFetchStep', [('Any SN WHERE NOT 5 in_state S, S name SN, S is State', - [{'S': 'State', 'SN': 'String'}])], + [{'S': 'State', 'SN': 'String'}])], None, None, [self.rql, self.system], {}, [])], {'x': ueid}) @@ -1231,10 +1231,10 @@ repo._type_source_cache[999999] = ('Note', 'cards', 999999) # similar to the above test but with an eid coming from the external source. # the same plan may be used, since we won't find any record in the system source - # linking 9999999 to a state + # linking 9999999 to a state self._test('Any SN WHERE NOT X in_state S, X eid %(x)s, S name SN', [('OneFetchStep', [('Any SN WHERE NOT 999999 in_state S, S name SN, S is State', - [{'S': 'State', 'SN': 'String'}])], + [{'S': 'State', 'SN': 'String'}])], None, None, [self.rql, self.system], {}, [])], {'x': 999999}) @@ -1257,7 +1257,7 @@ None, None, [self.system], {'S': 'table0.C1', 'S.name': 'table0.C0', 'SN': 'table0.C0'}, []),] )]) - + def test_external_attributes_and_relation(self): repo._type_source_cache[999999] = ('Note', 'cards', 999999) self._test('Any A,B,C,D WHERE A eid %(x)s,A creation_date B,A modification_date C, A todo_by D?', @@ -1287,7 +1287,7 @@ [('OneFetchStep', [('Any X WHERE X has_text "toto", X is Card', [{'X': 'Card'}])], None, None, [self.system], {}, [])]) - + def test_has_text_3(self): self._test('Any X WHERE X has_text "toto", X title "zoubidou"', [('FetchStep', [(u'Any X WHERE X title "zoubidou", X is Card', @@ -1302,7 +1302,7 @@ None, None, [self.system], {}, []), ]), ]) - + def test_sort_func(self): self._test('Note X ORDERBY DUMB_SORT(RF) WHERE X type RF', [('AggrStep', 'Any X ORDERBY DUMB_SORT(RF)', None, None, 'table0', None, [ @@ -1362,7 +1362,7 @@ def test_attr_unification_neq_1(self): self._test('Any X,Y WHERE X is Bookmark, Y is Card, X creation_date D, Y creation_date > D', - [('FetchStep', + [('FetchStep', [('Any Y,D WHERE Y creation_date > D, Y is Card', [{'D': 'Datetime', 'Y': 'Card'}])], [self.rql,self.system], None, @@ -1439,7 +1439,7 @@ # external source w/ .cross_relations == ['multisource_crossed_rel'] ###### - + def test_crossed_relation_eid_1_invariant(self): repo._type_source_cache[999999] = ('Note', 'system', 999999) self._test('Any Y WHERE X eid %(x)s, X multisource_crossed_rel Y', @@ -1471,7 +1471,7 @@ def test_crossed_relation_eid_2_needattr(self): repo._type_source_cache[999999] = ('Note', 'cards', 999999) - self._test('Any Y,T WHERE X eid %(x)s, X multisource_crossed_rel Y, Y type T', + self._test('Any Y,T WHERE X eid %(x)s, X multisource_crossed_rel Y, Y type T', [('FetchStep', [('Any Y,T WHERE Y type T, Y is Note', [{'T': 'String', 'Y': 'Note'}])], [self.rql, self.system], None, {'T': 'table0.C1', 'Y': 'table0.C0', 'Y.type': 'table0.C1'}, []), @@ -1527,7 +1527,7 @@ [])] )], {'x': 999999,}) - + # edition queries tests ################################################### def test_insert_simplified_var_1(self): @@ -1602,7 +1602,7 @@ )] )], {'n': 999999, 's': 999998}) - + def test_delete_relation1(self): ueid = self.session.user.eid self._test('DELETE X created_by Y WHERE X eid %(x)s, NOT Y eid %(y)s', @@ -1613,7 +1613,7 @@ ]), ], {'x': ueid, 'y': ueid}) - + def test_delete_relation2(self): ueid = self.session.user.eid self._test('DELETE X created_by Y WHERE X eid %(x)s, NOT Y login "syt"', @@ -1636,7 +1636,7 @@ ]) ], {'x': 999999}) - + def test_delete_entity2(self): repo._type_source_cache[999999] = ('Note', 'system', 999999) self._test('DELETE Note X WHERE X eid %(x)s, NOT X multisource_inlined_rel Y', @@ -1647,7 +1647,7 @@ ]) ], {'x': 999999}) - + def test_update(self): self._test('SET X copain Y WHERE X login "comme", Y login "cochon"', [('FetchStep', @@ -1701,9 +1701,9 @@ # None, None, [self.system], {}, []), # ]), # ]) - + # non regression tests #################################################### - + def test_nonregr1(self): self._test('Any X, Y WHERE X copain Y, X login "syt", Y login "cochon"', [('FetchStep', @@ -1717,7 +1717,7 @@ [{'X': 'CWUser', 'Y': 'CWUser'}])], None, None, [self.system], {'X': 'table0.C0', 'Y': 'table1.C0'}, []) ]) - + def test_nonregr2(self): treid = self.session.user.latest_trinfo().eid self._test('Any X ORDERBY D DESC WHERE E eid %(x)s, E wf_info_for X, X modification_date D', @@ -1746,7 +1746,7 @@ ]), ], {'x': treid}) - + def test_nonregr3(self): # original jpl query: # Any X, NOW - CD, P WHERE P is Project, U interested_in P, U is CWUser, U login "sthenault", X concerns P, X creation_date CD ORDERBY CD DESC LIMIT 5 @@ -1757,7 +1757,7 @@ [{'P': 'Bookmark', 'U': 'CWUser', 'X': 'CWEType', 'CD': 'Datetime'}])], 5, None, [self.system], {'U': 'table0.C0'}, [])] ) - + def test_nonregr4(self): self._test('Any U ORDERBY D DESC WHERE WF wf_info_for X, WF creation_date D, WF from_state FS, ' 'WF owned_by U?, X eid %(x)s', @@ -1771,7 +1771,7 @@ def test_nonregr5(self): # original jpl query: - # DISTINCT Version V WHERE MB done_in MV, MV eid %(x)s, + # DISTINCT Version V WHERE MB done_in MV, MV eid %(x)s, # MB depends_on B, B done_in V, V version_of P, NOT P eid %(p)s' cardeid = self.execute('INSERT Card X: X title "hop"')[0][0] noteeid = self.execute('INSERT Note X')[0][0] @@ -1822,7 +1822,7 @@ [{'Z': 'Affaire'}])], None, None, [self.system], {}, [])], {'x': 999999}) - + def test_nonregr9(self): repo._type_source_cache[999999] = ('Note', 'cards', 999999) repo._type_source_cache[999998] = ('Note', 'cards', 999998) @@ -1853,7 +1853,7 @@ []) ], {'x': 999999}) - + def test_nonregr11(self): repo._type_source_cache[999999] = ('Bookmark', 'system', 999999) self._test('SET X bookmarked_by Y WHERE X eid %(x)s, Y login "hop"', @@ -1867,7 +1867,7 @@ [])] )], {'x': 999999}) - + def test_nonregr12(self): repo._type_source_cache[999999] = ('Note', 'cards', 999999) self._test('Any X ORDERBY Z DESC WHERE X modification_date Z, E eid %(x)s, E see_also X', @@ -1916,7 +1916,7 @@ {'U': 'table1.C0', 'UL': 'table1.C1'}, [])], {'x': self.session.user.eid}) - + def test_nonregr13_2(self): # identity *not* wrapped into exists. # @@ -1954,11 +1954,11 @@ class MSPlannerTwoSameExternalSourcesTC(BasePlannerTC): """test planner related feature on a 3-sources repository: - + * 2 rql sources supporting Card """ repo = repo - + def setUp(self): self.o = repo.querier self.session = repo._sessions.values()[0] @@ -1982,7 +1982,7 @@ assert 'multisource_crossed_rel' in repo.sources_by_uri['cards'].cross_relations clear_ms_caches(repo) _test = test_plan - + def tearDown(self): undo_monkey_patch() del self.sources[-1] @@ -2090,6 +2090,16 @@ )] ) + def test_nonregr_dont_cross_rel_source_filtering(self): + self.repo._type_source_cache[999999] = ('Note', 'cards', 999999) + self._test('Any X,AA,AB WHERE E eid %(x)s, E in_state X, X name AA, X modification_date AB', + [('OneFetchStep', [('Any X,AA,AB WHERE 999999 in_state X, X name AA, X modification_date AB, X is State', + [{'AA': 'String', 'AB': 'Datetime', 'X': 'State'}])], + None, None, + [self.rql], {}, [] + )], + {'x': 999999}) + if __name__ == '__main__': from logilab.common.testlib import unittest_main