--- a/server/msplanner.py Wed May 13 15:06:02 2009 +0200
+++ b/server/msplanner.py Wed May 13 15:46:47 2009 +0200
@@ -16,7 +16,7 @@
"cross_relations" set in the source's mapping file and it that case, we'll
consider that we can also find in the system source some relation between
X and Y coming from different sources.
-
+
* if "relation" isn't supported by the external source but X or Y
types (or both) are, we suppose by default that can find in the system
source some relation where X and/or Y come from the external source. You
@@ -49,7 +49,7 @@
1. return the result of CWUser X WHERE X in_group G, G name 'users' from system
source, that's enough (optimization of the sql querier will avoid join on
CWUser, so we will directly get local eids)
-
+
:CWUser X,L WHERE X in_group G, X login L, G name 'users':
1. fetch Any X,L WHERE X is CWUser, X login L from both sources, store
concatenation of results into a temporary table
@@ -98,7 +98,7 @@
AbstractSource.dont_cross_relations = ()
AbstractSource.cross_relations = ()
-
+
def need_aggr_step(select, sources, stepdefs=None):
"""return True if a temporary table is necessary to store some partial
results to execute the given query
@@ -169,7 +169,7 @@
for part in subparts:
newnode.append(part)
return newnode
-
+
def same_scope(var):
"""return true if the variable is always used in the same scope"""
try:
@@ -181,7 +181,7 @@
return False
var.stinfo['samescope'] = True
return True
-
+
################################################################################
class PartPlanInformation(object):
@@ -198,19 +198,19 @@
the execution plan
:attr rqlst:
the original rql syntax tree handled by this part
-
+
:attr needsplit:
bool telling if the query has to be split into multiple steps for
execution or if it can be executed at once
-
+
:attr temptable:
a SQL temporary table name or None, if necessary to handle aggregate /
sorting for this part of the query
-
+
:attr finaltable:
a SQL table name or None, if results for this part of the query should be
written into a temporary table (usually shared by multiple PPI)
-
+
:attr sourcesterms:
a dictionary {source : {term: set([solution index, ])}} telling for each
source which terms are supported for which solutions. A "term" may be
@@ -262,23 +262,23 @@
print 'sourcesterms:'
for source, terms in self.sourcesterms.items():
print source, terms
-
+
def copy_solutions(self, solindices):
return [self._solutions[solidx].copy() for solidx in solindices]
-
+
@property
@cached
def part_sources(self):
if self._sourcesterms:
return tuple(sorted(self._sourcesterms))
return (self.system_source,)
-
+
@property
@cached
def _sys_source_set(self):
return frozenset((self.system_source, solindex)
- for solindex in self._solindices)
-
+ for solindex in self._solindices)
+
@cached
def _norel_support_set(self, relation):
"""return a set of (source, solindex) where source doesn't support the
@@ -340,7 +340,7 @@
# query
if not varobj._q_invariant and any(ifilterfalse(
source.support_relation, (r.r_type for r in rels))):
- self.needsplit = True
+ self.needsplit = True
# add source for rewritten constants to sourcesterms
for vconsts in self.rqlst.stinfo['rewritten'].itervalues():
const = vconsts[0]
@@ -397,7 +397,7 @@
self._linkedterms.setdefault(lhsv, set()).add((rhsv, rel))
self._linkedterms.setdefault(rhsv, set()).add((lhsv, rel))
return termssources
-
+
def _handle_cross_relation(self, rel, relsources, termssources):
for source in relsources:
if rel.r_type in source.cross_relations:
@@ -422,7 +422,7 @@
break
else:
self._sourcesterms.setdefault(source, {})[rel] = set(self._solindices)
-
+
def _remove_invalid_sources(self, termssources):
"""removes invalid sources from `sourcesterms` member according to
traversed relations and their properties (which sources support them,
@@ -455,7 +455,7 @@
continue
self._remove_term_sources(lhsv, rel, rhsv, termssources)
self._remove_term_sources(rhsv, rel, lhsv, termssources)
-
+
def _extern_term(self, term, termssources, inserted):
var = term.variable
if var.stinfo['constnode']:
@@ -471,7 +471,7 @@
if not termv in termssources:
termssources[termv] = self._term_sources(termv)
return termv
-
+
def _remove_sources_until_stable(self, term, termssources):
sourcesterms = self._sourcesterms
for oterm, rel in self._linkedterms.get(term, ()):
@@ -506,10 +506,10 @@
self._remove_term_sources(term, rel, oterm, termssources)
if not need_ancestor_scope or is_ancestor(oterm.scope, term.scope):
self._remove_term_sources(oterm, rel, term, termssources)
-
+
def _remove_term_sources(self, term, rel, oterm, termssources):
"""remove invalid sources for term according to oterm's sources and the
- relation between those two terms.
+ relation between those two terms.
"""
norelsup = self._norel_support_set(rel)
termsources = termssources[term]
@@ -528,21 +528,23 @@
self._remove_sources(term, invalid_sources)
termsources -= invalid_sources
self._remove_sources_until_stable(term, termssources)
-
+ if isinstance(oterm, Constant):
+ self._remove_sources(oterm, invalid_sources)
+
def _compute_needsplit(self):
"""tell according to sourcesterms if the rqlst has to be splitted for
execution among multiple sources
-
+
the execution has to be split if
* a source support an entity (non invariant) but doesn't support a
relation on it
* a source support an entity which is accessed by an optional relation
- * there is more than one source and either all sources'supported
+ * there is more than one source and either all sources'supported
variable/solutions are not equivalent or multiple variables have to
be fetched from some source
"""
# NOTE: < 2 since may be 0 on queries such as Any X WHERE X eid 2
- if len(self._sourcesterms) < 2:
+ if len(self._sourcesterms) < 2:
self.needsplit = False
elif not self.needsplit:
if not allequals(self._sourcesterms.itervalues()):
@@ -576,7 +578,7 @@
if not r is rel and self._repo.is_multi_sources_relation(r.r_type)):
return True
return False
-
+
def _set_source_for_term(self, source, term):
self._sourcesterms.setdefault(source, {})[term] = set(self._solindices)
@@ -603,7 +605,7 @@
try:
sourcesterms[source][term].remove(solindex)
except KeyError:
- return # may occur with subquery column alias
+ continue # may occur with subquery column alias
if not sourcesterms[source][term]:
del sourcesterms[source][term]
if not sourcesterms[source]:
@@ -611,7 +613,7 @@
def crossed_relation(self, source, relation):
return relation in self._crossrelations.get(source, ())
-
+
def part_steps(self):
"""precompute necessary part steps before generating actual rql for
each step. This is necessary to know if an aggregate step will be
@@ -758,10 +760,10 @@
# ensure relation is using '=' operator, else we rely on a
# sqlgenerator side effect (it won't insert an inequality operator
# in this case)
- relation.children[1].operator = '='
+ relation.children[1].operator = '='
terms.append(newvar)
needsel.add(newvar.name)
-
+
def _choose_term(self, sourceterms):
"""pick one term among terms supported by a source, which will be used
as a base to generate an execution step
@@ -798,7 +800,7 @@
# whatever (relation)
term = iter(sourceterms).next()
return term, sourceterms.pop(term)
-
+
def _expand_sources(self, selected_source, term, solindices):
"""return all sources supporting given term / solindices"""
sources = [selected_source]
@@ -806,7 +808,7 @@
for source in sourcesterms.keys():
if source is selected_source:
continue
- if not (term in sourcesterms[source] and
+ if not (term in sourcesterms[source] and
solindices.issubset(sourcesterms[source][term])):
continue
sources.append(source)
@@ -818,7 +820,7 @@
if not sourcesterms[source]:
del sourcesterms[source]
return sources
-
+
def _expand_terms(self, term, sources, sourceterms, scope, solindices):
terms = [term]
sources = sorted(sources)
@@ -876,7 +878,7 @@
modified = True
self._cleanup_sourcesterms(sources, solindices, term)
return terms
-
+
def _cleanup_sourcesterms(self, sources, solindices, term=None):
"""remove solutions so we know they are already processed"""
for source in sources:
@@ -901,7 +903,7 @@
#assert term in cross_terms
if not sourceterms:
del self._sourcesterms[source]
-
+
def merge_input_maps(self, allsolindices):
"""inputmaps is a dictionary with tuple of solution indices as key with
an associated input map as value. This function compute for each
@@ -911,7 +913,7 @@
inputmaps = {(0, 1, 2): {'A': 't1.login1', 'U': 't1.C0', 'U.login': 't1.login1'},
(1,): {'X': 't2.C0', 'T': 't2.C1'}}
return : [([1], {'A': 't1.login1', 'U': 't1.C0', 'U.login': 't1.login1',
- 'X': 't2.C0', 'T': 't2.C1'}),
+ 'X': 't2.C0', 'T': 't2.C1'}),
([0,2], {'A': 't1.login1', 'U': 't1.C0', 'U.login': 't1.login1'})]
"""
if not self._inputmaps:
@@ -980,10 +982,10 @@
decompose the RQL query according to sources'schema
"""
-
+
def build_select_plan(self, plan, rqlst):
"""build execution plan for a SELECT RQL query
-
+
the rqlst should not be tagged at this point
"""
if server.DEBUG:
@@ -1030,7 +1032,7 @@
inputmap[colalias.name] = '%s.C%s' % (temptable, i)
ppi.plan.add_step(sstep)
return inputmap
-
+
def _union_plan(self, plan, union, ppis, temptable=None):
tosplit, cango, allsources = [], {}, set()
for planinfo in ppis:
@@ -1088,7 +1090,7 @@
return steps
# internal methods for multisources decomposition #########################
-
+
def split_part(self, ppi, temptable):
ppi.finaltable = temptable
plan = ppi.plan
@@ -1172,7 +1174,7 @@
step.set_limit_offset(select.limit, select.offset)
return step
-
+
class UnsupportedBranch(Exception):
pass
@@ -1185,7 +1187,7 @@
self.hasaggrstep = self.ppi.temptable
self.extneedsel = frozenset(vref.name for sortterm in ppi.rqlst.orderby
for vref in sortterm.iget_nodes(VariableRef))
-
+
def _rqlst_accept(self, rqlst, node, newroot, terms, setfunc=None):
try:
newrestr, node_ = node.accept(self, newroot, terms[:])
@@ -1293,7 +1295,7 @@
if server.DEBUG:
print '--->', newroot
return newroot, self.insertedvars
-
+
def visit_and(self, node, newroot, terms):
subparts = []
for i in xrange(len(node.children)):
@@ -1330,7 +1332,7 @@
if termsources and termsources != self.sources:
return False
return True
-
+
def visit_relation(self, node, newroot, terms):
if not node.is_types_restriction():
if node in self.skip and self.solindices.issubset(self.skip[node]):
@@ -1368,7 +1370,7 @@
if not ored:
self.skip.setdefault(node, set()).update(self.solindices)
else:
- self.mayneedvar.setdefault((node.children[0].name, rschema), []).append( (res, ored) )
+ self.mayneedvar.setdefault((node.children[0].name, rschema), []).append( (res, ored) )
else:
assert len(vrefs) == 1
vref = vrefs[0]
@@ -1391,7 +1393,7 @@
if any(v for v, _ in var.stinfo['attrvars'] if not v in terms):
return False
return True
-
+
def visit_exists(self, node, newroot, terms):
newexists = node.__class__()
self.scopes = {node: newexists}
@@ -1400,18 +1402,18 @@
return None, node
newexists.set_where(subparts[0])
return newexists, node
-
+
def visit_not(self, node, newroot, terms):
subparts, node = self._visit_children(node, newroot, terms)
if not subparts:
return None, node
return copy_node(newroot, node, subparts), node
-
+
def visit_group(self, node, newroot, terms):
if not self.final:
return None, node
return self.visit_default(node, newroot, terms)
-
+
def visit_variableref(self, node, newroot, terms):
if self.use_only_defined:
if not node.variable.name in newroot.defined_vars:
@@ -1426,14 +1428,14 @@
def visit_constant(self, node, newroot, terms):
return copy_node(newroot, node), node
-
+
def visit_default(self, node, newroot, terms):
subparts, node = self._visit_children(node, newroot, terms)
return copy_node(newroot, node, subparts), node
-
+
visit_comparison = visit_mathexpression = visit_constant = visit_function = visit_default
visit_sort = visit_sortterm = visit_default
-
+
def _visit_children(self, node, newroot, terms):
subparts = []
for i in xrange(len(node.children)):
@@ -1444,14 +1446,14 @@
if newchild is not None:
subparts.append(newchild)
return subparts, node
-
+
def process_selection(self, newroot, terms, rqlst):
if self.final:
for term in rqlst.selection:
newroot.append_selected(term.copy(newroot))
for vref in term.get_nodes(VariableRef):
self.needsel.add(vref.name)
- return
+ return
for term in rqlst.selection:
vrefs = term.get_nodes(VariableRef)
if vrefs:
@@ -1471,7 +1473,7 @@
for vref in supportedvars:
if not vref in newroot.get_selected_variables():
newroot.append_selected(VariableRef(newroot.get_variable(vref.name)))
-
+
def add_necessary_selection(self, newroot, terms):
selected = tuple(newroot.get_selected_variables())
for varname in terms:
--- a/server/test/unittest_msplanner.py Wed May 13 15:06:02 2009 +0200
+++ b/server/test/unittest_msplanner.py Wed May 13 15:46:47 2009 +0200
@@ -24,15 +24,15 @@
def syntax_tree_search(self, *args, **kwargs):
return []
-
+
class FakeCardSource(AbstractSource):
uri = 'ccc'
support_entities = {'Card': True, 'Note': True, 'State': True}
support_relations = {'in_state': True, 'multisource_rel': True, 'multisource_inlined_rel': True,
'multisource_crossed_rel': True}
- dont_cross_relations = set(('fiche',))
+ dont_cross_relations = set(('fiche', 'in_state'))
cross_relations = set(('multisource_crossed_rel',))
-
+
def syntax_tree_search(self, *args, **kwargs):
return []
@@ -53,19 +53,19 @@
clear_cache(repo, 'can_cross_relation')
clear_cache(repo, 'is_multi_sources_relation')
# XXX source_defs
-
+
# keep cnx so it's not garbage collected and the associated session is closed
repo, cnx = init_test_database('sqlite')
class BaseMSPlannerTC(BasePlannerTC):
"""test planner related feature on a 3-sources repository:
-
+
* system source supporting everything
* ldap source supporting CWUser
* rql source supporting Card
"""
repo = repo
-
+
def setUp(self):
#_QuerierTC.setUp(self)
clear_cache(repo, 'rel_type_sources')
@@ -84,7 +84,7 @@
self.prevrqlexpr_user = userreadperms[-1]
userreadperms[-1] = ERQLExpression('X owned_by U')
self.schema['CWUser']._groups['read'] = tuple(userreadperms)
-
+
self.sources = self.o._repo.sources
self.system = self.sources[-1]
self.sources.append(FakeUserROSource(self.o._repo, self.o.schema,
@@ -97,7 +97,7 @@
self.rql = self.sources[-1]
do_monkey_patch()
clear_ms_caches(repo)
-
+
def tearDown(self):
undo_monkey_patch()
del self.sources[-1]
@@ -107,20 +107,20 @@
# restore hijacked security
self.restore_orig_affaire_security()
self.restore_orig_euser_security()
-
+
def restore_orig_affaire_security(self):
affreadperms = list(self.schema['Affaire']._groups['read'])
affreadperms[-1] = self.prevrqlexpr_affaire
self.schema['Affaire']._groups['read'] = tuple(affreadperms)
clear_cache(self.schema['Affaire'], 'ERSchema_get_rqlexprs')
-
+
def restore_orig_euser_security(self):
userreadperms = list(self.schema['CWUser']._groups['read'])
userreadperms[-1] = self.prevrqlexpr_user
self.schema['CWUser']._groups['read'] = tuple(userreadperms)
clear_cache(self.schema['CWUser'], 'ERSchema_get_rqlexprs')
-
+
class PartPlanInformationTC(BaseMSPlannerTC):
def _test(self, rql, *args):
@@ -140,44 +140,44 @@
self.assertEquals(ppi._sourcesterms, sourcesterms)
self.assertEquals(ppi.needsplit, needsplit)
-
+
def test_simple_system_only(self):
"""retrieve entities only supported by the system source"""
self._test('CWGroup X',
{self.system: {'X': s[0]}}, False)
-
+
def test_simple_system_ldap(self):
"""retrieve CWUser X from both sources and return concatenation of results
"""
self._test('CWUser X',
{self.system: {'X': s[0]}, self.ldap: {'X': s[0]}}, False)
-
+
def test_simple_system_rql(self):
"""retrieve Card X from both sources and return concatenation of results
"""
self._test('Any X, XT WHERE X is Card, X title XT',
{self.system: {'X': s[0]}, self.rql: {'X': s[0]}}, False)
-
+
def test_simple_eid_specified(self):
"""retrieve CWUser X from system source (eid is specified, can locate the entity)
"""
ueid = self.session.user.eid
self._test('Any X,L WHERE X eid %(x)s, X login L', {'x': ueid},
{self.system: {'X': s[0]}}, False)
-
+
def test_simple_eid_invariant(self):
"""retrieve CWUser X from system source (eid is specified, can locate the entity)
"""
ueid = self.session.user.eid
self._test('Any X WHERE X eid %(x)s', {'x': ueid},
{self.system: {'x': s[0]}}, False)
-
+
def test_simple_invariant(self):
"""retrieve CWUser X from system source only (X is invariant and in_group not supported by ldap source)
"""
self._test('Any X WHERE X is CWUser, X in_group G, G name "users"',
{self.system: {'X': s[0], 'G': s[0], 'in_group': s[0]}}, False)
-
+
def test_security_has_text(self):
"""retrieve CWUser X from system source only (has_text not supported by ldap source)
"""
@@ -185,7 +185,7 @@
# with ambigous query (eg only considering the first solution)
self._test('CWUser X WHERE X has_text "bla"',
{self.system: {'X': s[0]}}, False)
-
+
def test_complex_base(self):
"""
1. retrieve Any X, L WHERE X is CWUser, X login L from system and ldap sources, store
@@ -202,7 +202,7 @@
1. retrieve Any X,AA WHERE X modification_date AA from system and ldap sources, store
concatenation of results into a temporary table
2. return the result of Any X,AA ORDERBY AA WHERE %s owned_by X, X modification_date AA
- on the system source
+ on the system source
"""
ueid = self.session.user.eid
self._test('Any X,AA ORDERBY AA WHERE E eid %(x)s, E owned_by X, X modification_date AA', {'x': ueid},
@@ -214,7 +214,7 @@
1. retrieve Any X,L,AA WHERE X login L, X modification_date AA from system and ldap sources, store
concatenation of results into a temporary table
2. return the result of Any X,L,AA WHERE %s owned_by X, X login L, X modification_date AA
- on the system source
+ on the system source
"""
ueid = self.session.user.eid
self._test('Any X,L,AA WHERE E eid %(x)s, E owned_by X, X login L, X modification_date AA', {'x': ueid},
@@ -233,18 +233,18 @@
1. retrieve Any X,A,Y,B WHERE X login A, Y login B from system and ldap sources, store
cartesian product of results into a temporary table
2. return the result of Any X,Y WHERE X login 'syt', Y login 'adim'
- on the system source
+ on the system source
"""
ueid = self.session.user.eid
self._test('Any X,Y WHERE X login "syt", Y login "adim"', {'x': ueid},
{self.system: {'Y': s[0], 'X': s[0]},
self.ldap: {'Y': s[0], 'X': s[0]}}, True)
-
+
def test_complex_aggregat(self):
solindexes = set(range(len([e for e in self.schema.entities() if not e.is_final()])))
self._test('Any MAX(X)',
{self.system: {'X': solindexes}}, False)
-
+
def test_complex_optional(self):
ueid = self.session.user.eid
self._test('Any U WHERE WF wf_info_for X, X eid %(x)s, WF owned_by U?, WF from_state FS', {'x': ueid},
@@ -252,7 +252,7 @@
'from_state': s[0], 'owned_by': s[0], 'wf_info_for': s[0],
'x': s[0]}},
False)
-
+
def test_exists4(self):
"""
State S could come from both rql source and system source,
@@ -262,7 +262,7 @@
self._test('Any G,L WHERE X in_group G, X login L, G name "managers", '
'EXISTS(X copain T, T login L, T login in ("comme", "cochon")) OR '
'EXISTS(X in_state S, S name "pascontent", NOT X copain T2, T2 login "billy")',
- {self.system: {'X': s[0], 'S': s[0], 'T2': s[0], 'T': s[0], 'G': s[0], 'copain': s[0], 'in_group': s[0]},
+ {self.system: {'X': s[0], 'S': s[0], 'T2': s[0], 'T': s[0], 'G': s[0], 'copain': s[0], 'in_group': s[0]},
self.ldap: {'X': s[0], 'T2': s[0], 'T': s[0]}},
True)
@@ -271,18 +271,18 @@
{self.system: {'X': s[0, 1, 2], 'S': s[0, 1, 2]},
self.rql: {'X': s[2], 'S': s[2]}},
True)
-
+
def test_not_relation_need_split(self):
self._test('Any SN WHERE NOT X in_state S, S name SN',
{self.rql: {'X': s[2], 'S': s[0, 1, 2]},
self.system: {'X': s[0, 1, 2], 'S': s[0, 1, 2]}},
True)
-
+
def test_not_relation_no_split_external(self):
repo._type_source_cache[999999] = ('Note', 'cards', 999999)
# similar to the above test but with an eid coming from the external source.
# the same plan may be used, since we won't find any record in the system source
- # linking 9999999 to a state
+ # linking 9999999 to a state
self._test('Any SN WHERE NOT X in_state S, X eid %(x)s, S name SN',
{'x': 999999},
{self.rql: {'x': s[0], 'S': s[0]},
@@ -303,7 +303,7 @@
'require_permission': s[0], 'in_group': s[0], 'P': s[0], 'require_group': s[0],
'u': s[0]}},
False)
-
+
def test_delete_relation1(self):
ueid = self.session.user.eid
self._test('Any X, Y WHERE X created_by Y, X eid %(x)s, NOT Y eid %(y)s',
@@ -318,7 +318,7 @@
{'x': 999999,},
{self.rql: {'Y': s[0]}, self.system: {'Y': s[0], 'x': s[0]}},
True)
-
+
def test_crossed_relation_eid_1_invariant(self):
repo._type_source_cache[999999] = ('Note', 'system', 999999)
self._test('Any Y WHERE X eid %(x)s, X multisource_crossed_rel Y',
@@ -341,7 +341,7 @@
{self.rql: {'X': s[0], 'AD': s[0], 'multisource_crossed_rel': s[0], 'x': s[0]},
self.system: {'X': s[0], 'AD': s[0], 'multisource_crossed_rel': s[0], 'x': s[0]}},
True)
-
+
def test_version_crossed_depends_on_2(self):
repo._type_source_cache[999999] = ('Note', 'system', 999999)
self._test('Any X,AD,AE WHERE E eid %(x)s, E multisource_crossed_rel X, X in_state AD, AD name AE',
@@ -356,11 +356,11 @@
self._test('Any S,T WHERE S eid %(s)s, N eid %(n)s, N type T, N is Note, S is State',
{'n': 999999, 's': 999998},
{self.rql: {'s': s[0], 'N': s[0]}}, False)
-
+
+
-
class MSPlannerTC(BaseMSPlannerTC):
-
+
def setUp(self):
BaseMSPlannerTC.setUp(self)
self.planner = MSPlanner(self.o.schema, self.o._rqlhelper)
@@ -387,14 +387,14 @@
self._test('CWGroup X LIMIT 10 OFFSET 10',
[('OneFetchStep', [('Any X LIMIT 10 OFFSET 10 WHERE X is CWGroup', [{'X': 'CWGroup'}])],
10, 10, [self.system], {}, [])])
-
+
def test_simple_system_ldap(self):
"""retrieve CWUser X from both sources and return concatenation of results
"""
self._test('CWUser X',
[('OneFetchStep', [('Any X WHERE X is CWUser', [{'X': 'CWUser'}])],
None, None, [self.ldap, self.system], {}, [])])
-
+
def test_simple_system_ldap_limit(self):
"""retrieve CWUser X from both sources and return concatenation of results
"""
@@ -428,14 +428,14 @@
[self.ldap, self.system], {}, {'COUNT(X)': 'table0.C0'}, []),
]),
])
-
+
def test_simple_system_rql(self):
"""retrieve Card X from both sources and return concatenation of results
"""
self._test('Any X, XT WHERE X is Card, X title XT',
[('OneFetchStep', [('Any X,XT WHERE X is Card, X title XT', [{'X': 'Card', 'XT': 'String'}])],
None, None, [self.rql, self.system], {}, [])])
-
+
def test_simple_eid_specified(self):
"""retrieve CWUser X from system source (eid is specified, can locate the entity)
"""
@@ -444,7 +444,7 @@
[('OneFetchStep', [('Any X,L WHERE X eid %s, X login L'%ueid, [{'X': 'CWUser', 'L': 'String'}])],
None, None, [self.system], {}, [])],
{'x': ueid})
-
+
def test_simple_eid_invariant(self):
"""retrieve CWUser X from system source (eid is specified, can locate the entity)
"""
@@ -453,7 +453,7 @@
[('OneFetchStep', [('Any %s'%ueid, [{}])],
None, None, [self.system], {}, [])],
{'x': ueid})
-
+
def test_simple_invariant(self):
"""retrieve CWUser X from system source only (X is invariant and in_group not supported by ldap source)
"""
@@ -461,7 +461,7 @@
[('OneFetchStep', [('Any X WHERE X is CWUser, X in_group G, G name "users"',
[{'X': 'CWUser', 'G': 'CWGroup'}])],
None, None, [self.system], {}, [])])
-
+
def test_complex_base(self):
"""
1. retrieve Any X, L WHERE X is CWUser, X login L from system and ldap sources, store
@@ -498,7 +498,7 @@
def test_complex_ordered(self):
self._test('Any L ORDERBY L WHERE X login L',
- [('AggrStep', 'Any L ORDERBY L', None, None, 'table0', None,
+ [('AggrStep', 'Any L ORDERBY L', None, None, 'table0', None,
[('FetchStep', [('Any L WHERE X login L, X is CWUser',
[{'X': 'CWUser', 'L': 'String'}])],
[self.ldap, self.system], {}, {'X.login': 'table0.C0', 'L': 'table0.C0'}, []),
@@ -507,13 +507,13 @@
def test_complex_ordered_limit_offset(self):
self._test('Any L ORDERBY L LIMIT 10 OFFSET 10 WHERE X login L',
- [('AggrStep', 'Any L ORDERBY L', 10, 10, 'table0', None,
+ [('AggrStep', 'Any L ORDERBY L', 10, 10, 'table0', None,
[('FetchStep', [('Any L WHERE X login L, X is CWUser',
[{'X': 'CWUser', 'L': 'String'}])],
[self.ldap, self.system], {}, {'X.login': 'table0.C0', 'L': 'table0.C0'}, []),
])
])
-
+
def test_complex_invariant_ordered(self):
"""
1. retrieve Any X,AA WHERE X modification_date AA from system and ldap sources, store
@@ -543,7 +543,7 @@
1. retrieve Any X,L,AA WHERE X login L, X modification_date AA from system and ldap sources, store
concatenation of results into a temporary table
2. return the result of Any X,L,AA WHERE %s owned_by X, X login L, X modification_date AA
- on the system source
+ on the system source
"""
ueid = self.session.user.eid
self._test('Any X,L,AA WHERE E eid %(x)s, E owned_by X, X login L, X modification_date AA',
@@ -593,7 +593,7 @@
2. return content of the table sorted
"""
self._test('Any X,F ORDERBY F WHERE X firstname F',
- [('AggrStep', 'Any X,F ORDERBY F', None, None, 'table0', None,
+ [('AggrStep', 'Any X,F ORDERBY F', None, None, 'table0', None,
[('FetchStep', [('Any X,F WHERE X firstname F, X is CWUser',
[{'X': 'CWUser', 'F': 'String'}])],
[self.ldap, self.system], {},
@@ -604,13 +604,13 @@
{'X': 'table0.C0', 'X.firstname': 'table0.C1', 'F': 'table0.C1'}, []),
]),
])
-
+
def test_complex_multiple(self):
"""
1. retrieve Any X,A,Y,B WHERE X login A, Y login B from system and ldap sources, store
cartesian product of results into a temporary table
2. return the result of Any X,Y WHERE X login 'syt', Y login 'adim'
- on the system source
+ on the system source
"""
ueid = self.session.user.eid
self._test('Any X,Y WHERE X login "syt", Y login "adim"',
@@ -627,13 +627,13 @@
None, None, [self.system],
{'X': 'table0.C0', 'Y': 'table1.C0'}, [])
], {'x': ueid})
-
+
def test_complex_multiple_limit_offset(self):
"""
1. retrieve Any X,A,Y,B WHERE X login A, Y login B from system and ldap sources, store
cartesian product of results into a temporary table
2. return the result of Any X,Y WHERE X login 'syt', Y login 'adim'
- on the system source
+ on the system source
"""
ueid = self.session.user.eid
self._test('Any X,Y LIMIT 10 OFFSET 10 WHERE X login "syt", Y login "adim"',
@@ -648,14 +648,14 @@
10, 10, [self.system],
{'X': 'table0.C0', 'Y': 'table1.C0'}, [])
], {'x': ueid})
-
+
def test_complex_aggregat(self):
self._test('Any MAX(X)',
[('OneFetchStep',
[('Any MAX(X)', X_ALL_SOLS)],
None, None, [self.system], {}, [])
])
-
+
def test_complex_typed_aggregat(self):
self._test('Any MAX(X) WHERE X is Card',
[('AggrStep', 'Any MAX(X)', None, None, 'table0', None,
@@ -664,21 +664,21 @@
[self.rql, self.system], {}, {'MAX(X)': 'table0.C0'}, [])
])
])
-
+
def test_complex_greater_eid(self):
self._test('Any X WHERE X eid > 12',
[('OneFetchStep',
[('Any X WHERE X eid > 12', X_ALL_SOLS)],
None, None, [self.system], {}, [])
])
-
+
def test_complex_greater_typed_eid(self):
self._test('Any X WHERE X eid > 12, X is Card',
[('OneFetchStep',
[('Any X WHERE X eid > 12, X is Card', [{'X': 'Card'}])],
None, None, [self.system], {}, [])
])
-
+
def test_complex_optional(self):
ueid = self.session.user.eid
self._test('Any U WHERE WF wf_info_for X, X eid %(x)s, WF owned_by U?, WF from_state FS',
@@ -695,7 +695,7 @@
None, None, [self.system], {}, [])],
{'x': ueid})
-
+
def test_3sources_ambigous(self):
self._test('Any X,T WHERE X owned_by U, U login "syt", X title T',
[('FetchStep', [('Any X,T WHERE X title T, X is Card', [{'X': 'Card', 'T': 'String'}])],
@@ -736,7 +736,7 @@
])
def test_outer_supported_rel1(self):
- # both system and rql support all variables, can be
+ # both system and rql support all variables, can be
self._test('Any X, R WHERE X is Note, X in_state S, X type R, '
'NOT EXISTS(Y is Note, Y in_state S, Y type R, X identity Y)',
[('OneFetchStep', [('Any X,R WHERE X is Note, X in_state S, X type R, NOT EXISTS(Y is Note, Y in_state S, Y type R, X identity Y), S is State',
@@ -746,7 +746,7 @@
])
def test_not_identity(self):
- # both system and rql support all variables, can be
+ # both system and rql support all variables, can be
self._test('Any X WHERE NOT X identity U, U eid %s' % self.session.user.eid,
[('OneFetchStep',
[('Any X WHERE NOT X identity 5, X is CWUser', [{'X': 'CWUser'}])],
@@ -769,7 +769,7 @@
None, None, [self.system],
{'A': 'table0.C0', 'X': 'table1.C0', 'X.login': 'table1.C1', 'R': 'table1.C1', 'Y.type': 'table0.C1'}, [])
])
-
+
def test_security_has_text(self):
# use a guest user
self.session = self._user_session()[1]
@@ -795,7 +795,7 @@
None, None, [self.system], {}, []),
])
])
-
+
def test_security_has_text_limit_offset(self):
# use a guest user
self.session = self._user_session()[1]
@@ -828,9 +828,9 @@
{'X': 'Folder'}, {'X': 'Image'}, {'X': 'Note'},
{'X': 'Personne'}, {'X': 'Societe'}, {'X': 'State'},
{'X': 'SubDivision'}, {'X': 'Tag'}, {'X': 'Transition'}])],
- 10, 10, [self.system], {'X': 'table0.C0'}, [])
+ 10, 10, [self.system], {'X': 'table0.C0'}, [])
])
-
+
def test_security_user(self):
"""a guest user trying to see another user: EXISTS(X owned_by U) is automatically inserted"""
# use a guest user
@@ -842,7 +842,7 @@
('OneFetchStep',
[('Any X WHERE EXISTS(X owned_by 5), X is CWUser', [{'X': 'CWUser'}])],
None, None, [self.system], {'X': 'table0.C0'}, [])])
-
+
def test_security_complex_has_text(self):
# use a guest user
self.session = self._user_session()[1]
@@ -879,18 +879,18 @@
self.session = self._user_session()[1]
self._test('Any MAX(X)',
[('FetchStep', [('Any E WHERE E type "X", E is Note', [{'E': 'Note'}])],
- [self.rql, self.system], None, {'E': 'table1.C0'}, []),
+ [self.rql, self.system], None, {'E': 'table1.C0'}, []),
('FetchStep', [('Any X WHERE X is CWUser', [{'X': 'CWUser'}])],
[self.ldap, self.system], None, {'X': 'table2.C0'}, []),
('UnionFetchStep', [
('FetchStep', [('Any X WHERE EXISTS(X owned_by 5), X is Basket', [{'X': 'Basket'}])],
- [self.system], {}, {'X': 'table0.C0'}, []),
+ [self.system], {}, {'X': 'table0.C0'}, []),
('UnionFetchStep',
[('FetchStep', [('Any X WHERE X is IN(Card, Note, State)',
[{'X': 'Card'}, {'X': 'Note'}, {'X': 'State'}])],
[self.rql, self.system], {}, {'X': 'table0.C0'}, []),
('FetchStep',
- [('Any X WHERE X is IN(Bookmark, Comment, Division, CWCache, CWConstraint, CWConstraintType, CWEType, CWAttribute, CWGroup, CWRelation, CWPermission, CWProperty, CWRType, Email, EmailAddress, EmailPart, EmailThread, File, Folder, Image, Personne, RQLExpression, Societe, SubDivision, Tag, TrInfo, Transition)',
+ [('Any X WHERE X is IN(Bookmark, CWAttribute, CWCache, CWConstraint, CWConstraintType, CWEType, CWGroup, CWPermission, CWProperty, CWRType, CWRelation, Comment, Division, Email, EmailAddress, EmailPart, EmailThread, File, Folder, Image, Personne, RQLExpression, Societe, SubDivision, Tag, TrInfo, Transition)',
sorted([{'X': 'Bookmark'}, {'X': 'Comment'}, {'X': 'Division'},
{'X': 'CWCache'}, {'X': 'CWConstraint'}, {'X': 'CWConstraintType'},
{'X': 'CWEType'}, {'X': 'CWAttribute'}, {'X': 'CWGroup'},
@@ -906,16 +906,16 @@
[self.system], {'X': 'table2.C0'}, {'X': 'table0.C0'}, []),
('FetchStep', [('Any X WHERE (EXISTS(X owned_by 5)) OR ((((EXISTS(D concerne C?, C owned_by 5, C type "X", X identity D, C is Division, D is Affaire)) OR (EXISTS(H concerne G?, G owned_by 5, G type "X", X identity H, G is SubDivision, H is Affaire))) OR (EXISTS(I concerne F?, F owned_by 5, F type "X", X identity I, F is Societe, I is Affaire))) OR (EXISTS(J concerne E?, E owned_by 5, X identity J, E is Note, J is Affaire))), X is Affaire',
[{'C': 'Division', 'E': 'Note', 'D': 'Affaire', 'G': 'SubDivision', 'F': 'Societe', 'I': 'Affaire', 'H': 'Affaire', 'J': 'Affaire', 'X': 'Affaire'}])],
- [self.system], {'E': 'table1.C0'}, {'X': 'table0.C0'}, []),
+ [self.system], {'E': 'table1.C0'}, {'X': 'table0.C0'}, []),
]),
('OneFetchStep', [('Any MAX(X)', X_ALL_SOLS)],
None, None, [self.system], {'X': 'table0.C0'}, [])
])
-
+
def test_security_complex_aggregat2(self):
# use a guest user
self.session = self._user_session()[1]
- self._test('Any ET, COUNT(X) GROUPBY ET ORDERBY ET WHERE X is ET',
+ self._test('Any ET, COUNT(X) GROUPBY ET ORDERBY ET WHERE X is ET',
[('FetchStep', [('Any X WHERE X is IN(Card, Note, State)',
[{'X': 'Card'}, {'X': 'Note'}, {'X': 'State'}])],
[self.rql, self.system], None, {'X': 'table1.C0'}, []),
@@ -939,7 +939,7 @@
[self.system], {'X': 'table3.C0'}, {'ET': 'table0.C0', 'X': 'table0.C1'}, []),
# extra UnionFetchStep could be avoided but has no cost, so don't care
('UnionFetchStep',
- [('FetchStep', [('Any ET,X WHERE X is ET, ET is CWEType, X is IN(Bookmark, Comment, Division, CWCache, CWConstraint, CWConstraintType, CWEType, CWAttribute, CWGroup, CWRelation, CWPermission, CWProperty, CWRType, Email, EmailAddress, EmailPart, EmailThread, File, Folder, Image, Personne, RQLExpression, Societe, SubDivision, Tag, TrInfo, Transition)',
+ [('FetchStep', [('Any ET,X WHERE X is ET, ET is CWEType, X is IN(Bookmark, CWAttribute, CWCache, CWConstraint, CWConstraintType, CWEType, CWGroup, CWPermission, CWProperty, CWRType, CWRelation, Comment, Division, Email, EmailAddress, EmailPart, EmailThread, File, Folder, Image, Personne, RQLExpression, Societe, SubDivision, Tag, TrInfo, Transition)',
[{'X': 'Bookmark', 'ET': 'CWEType'}, {'X': 'Comment', 'ET': 'CWEType'},
{'X': 'Division', 'ET': 'CWEType'}, {'X': 'CWCache', 'ET': 'CWEType'},
{'X': 'CWConstraint', 'ET': 'CWEType'}, {'X': 'CWConstraintType', 'ET': 'CWEType'},
@@ -1054,7 +1054,7 @@
10, 10, [self.system],
{'X': 'table0.C0', 'X.title': 'table0.C1', 'XT': 'table0.C1', 'U': 'table1.C0'}, [])
])
-
+
def test_exists_base(self):
self._test('Any X,L,S WHERE X in_state S, X login L, EXISTS(X in_group G, G name "bougloup")',
[('FetchStep', [('Any X,L WHERE X login L, X is CWUser', [{'X': 'CWUser', 'L': 'String'}])],
@@ -1128,7 +1128,7 @@
None, None, [self.system],
{'T': 'table0.C0', 'T2': 'table1.C0',
'X': 'table2.C1', 'X.login': 'table2.C0', 'L': 'table2.C0'}, [])])
-
+
def test_exists_security_no_invariant(self):
ueid = self.session.user.eid
self._test('Any X,AA,AB,AC,AD ORDERBY AA WHERE X is CWUser, X login AA, X firstname AB, X surname AC, X modification_date AD, A eid %(B)s, \
@@ -1159,10 +1159,10 @@
self._test('Any X, S WHERE X in_state S',
[('UnionStep', None, None, [
('OneFetchStep', [('Any X,S WHERE X in_state S, S is State, X is IN(Affaire, CWUser)',
- [{'X': 'Affaire', 'S': 'State'}, {'X': 'CWUser', 'S': 'State'}])],
+ [{'X': 'Affaire', 'S': 'State'}, {'X': 'CWUser', 'S': 'State'}])],
None, None, [self.system], {}, []),
('OneFetchStep', [('Any X,S WHERE X in_state S, S is State, X is Note',
- [{'X': 'Note', 'S': 'State'}])],
+ [{'X': 'Note', 'S': 'State'}])],
None, None, [self.rql, self.system], {}, []),
])])
@@ -1223,7 +1223,7 @@
# generation for the external source
self._test('Any SN WHERE NOT X in_state S, X eid %(x)s, S name SN',
[('OneFetchStep', [('Any SN WHERE NOT 5 in_state S, S name SN, S is State',
- [{'S': 'State', 'SN': 'String'}])],
+ [{'S': 'State', 'SN': 'String'}])],
None, None, [self.rql, self.system], {}, [])],
{'x': ueid})
@@ -1231,10 +1231,10 @@
repo._type_source_cache[999999] = ('Note', 'cards', 999999)
# similar to the above test but with an eid coming from the external source.
# the same plan may be used, since we won't find any record in the system source
- # linking 9999999 to a state
+ # linking 9999999 to a state
self._test('Any SN WHERE NOT X in_state S, X eid %(x)s, S name SN',
[('OneFetchStep', [('Any SN WHERE NOT 999999 in_state S, S name SN, S is State',
- [{'S': 'State', 'SN': 'String'}])],
+ [{'S': 'State', 'SN': 'String'}])],
None, None, [self.rql, self.system], {}, [])],
{'x': 999999})
@@ -1257,7 +1257,7 @@
None, None, [self.system], {'S': 'table0.C1', 'S.name': 'table0.C0', 'SN': 'table0.C0'},
[]),]
)])
-
+
def test_external_attributes_and_relation(self):
repo._type_source_cache[999999] = ('Note', 'cards', 999999)
self._test('Any A,B,C,D WHERE A eid %(x)s,A creation_date B,A modification_date C, A todo_by D?',
@@ -1287,7 +1287,7 @@
[('OneFetchStep', [('Any X WHERE X has_text "toto", X is Card',
[{'X': 'Card'}])],
None, None, [self.system], {}, [])])
-
+
def test_has_text_3(self):
self._test('Any X WHERE X has_text "toto", X title "zoubidou"',
[('FetchStep', [(u'Any X WHERE X title "zoubidou", X is Card',
@@ -1302,7 +1302,7 @@
None, None, [self.system], {}, []),
]),
])
-
+
def test_sort_func(self):
self._test('Note X ORDERBY DUMB_SORT(RF) WHERE X type RF',
[('AggrStep', 'Any X ORDERBY DUMB_SORT(RF)', None, None, 'table0', None, [
@@ -1362,7 +1362,7 @@
def test_attr_unification_neq_1(self):
self._test('Any X,Y WHERE X is Bookmark, Y is Card, X creation_date D, Y creation_date > D',
- [('FetchStep',
+ [('FetchStep',
[('Any Y,D WHERE Y creation_date > D, Y is Card',
[{'D': 'Datetime', 'Y': 'Card'}])],
[self.rql,self.system], None,
@@ -1439,7 +1439,7 @@
# external source w/ .cross_relations == ['multisource_crossed_rel'] ######
-
+
def test_crossed_relation_eid_1_invariant(self):
repo._type_source_cache[999999] = ('Note', 'system', 999999)
self._test('Any Y WHERE X eid %(x)s, X multisource_crossed_rel Y',
@@ -1471,7 +1471,7 @@
def test_crossed_relation_eid_2_needattr(self):
repo._type_source_cache[999999] = ('Note', 'cards', 999999)
- self._test('Any Y,T WHERE X eid %(x)s, X multisource_crossed_rel Y, Y type T',
+ self._test('Any Y,T WHERE X eid %(x)s, X multisource_crossed_rel Y, Y type T',
[('FetchStep', [('Any Y,T WHERE Y type T, Y is Note', [{'T': 'String', 'Y': 'Note'}])],
[self.rql, self.system], None,
{'T': 'table0.C1', 'Y': 'table0.C0', 'Y.type': 'table0.C1'}, []),
@@ -1527,7 +1527,7 @@
[])]
)],
{'x': 999999,})
-
+
# edition queries tests ###################################################
def test_insert_simplified_var_1(self):
@@ -1602,7 +1602,7 @@
)]
)],
{'n': 999999, 's': 999998})
-
+
def test_delete_relation1(self):
ueid = self.session.user.eid
self._test('DELETE X created_by Y WHERE X eid %(x)s, NOT Y eid %(y)s',
@@ -1613,7 +1613,7 @@
]),
],
{'x': ueid, 'y': ueid})
-
+
def test_delete_relation2(self):
ueid = self.session.user.eid
self._test('DELETE X created_by Y WHERE X eid %(x)s, NOT Y login "syt"',
@@ -1636,7 +1636,7 @@
])
],
{'x': 999999})
-
+
def test_delete_entity2(self):
repo._type_source_cache[999999] = ('Note', 'system', 999999)
self._test('DELETE Note X WHERE X eid %(x)s, NOT X multisource_inlined_rel Y',
@@ -1647,7 +1647,7 @@
])
],
{'x': 999999})
-
+
def test_update(self):
self._test('SET X copain Y WHERE X login "comme", Y login "cochon"',
[('FetchStep',
@@ -1701,9 +1701,9 @@
# None, None, [self.system], {}, []),
# ]),
# ])
-
+
# non regression tests ####################################################
-
+
def test_nonregr1(self):
self._test('Any X, Y WHERE X copain Y, X login "syt", Y login "cochon"',
[('FetchStep',
@@ -1717,7 +1717,7 @@
[{'X': 'CWUser', 'Y': 'CWUser'}])],
None, None, [self.system], {'X': 'table0.C0', 'Y': 'table1.C0'}, [])
])
-
+
def test_nonregr2(self):
treid = self.session.user.latest_trinfo().eid
self._test('Any X ORDERBY D DESC WHERE E eid %(x)s, E wf_info_for X, X modification_date D',
@@ -1746,7 +1746,7 @@
]),
],
{'x': treid})
-
+
def test_nonregr3(self):
# original jpl query:
# Any X, NOW - CD, P WHERE P is Project, U interested_in P, U is CWUser, U login "sthenault", X concerns P, X creation_date CD ORDERBY CD DESC LIMIT 5
@@ -1757,7 +1757,7 @@
[{'P': 'Bookmark', 'U': 'CWUser', 'X': 'CWEType', 'CD': 'Datetime'}])],
5, None, [self.system], {'U': 'table0.C0'}, [])]
)
-
+
def test_nonregr4(self):
self._test('Any U ORDERBY D DESC WHERE WF wf_info_for X, WF creation_date D, WF from_state FS, '
'WF owned_by U?, X eid %(x)s',
@@ -1771,7 +1771,7 @@
def test_nonregr5(self):
# original jpl query:
- # DISTINCT Version V WHERE MB done_in MV, MV eid %(x)s,
+ # DISTINCT Version V WHERE MB done_in MV, MV eid %(x)s,
# MB depends_on B, B done_in V, V version_of P, NOT P eid %(p)s'
cardeid = self.execute('INSERT Card X: X title "hop"')[0][0]
noteeid = self.execute('INSERT Note X')[0][0]
@@ -1822,7 +1822,7 @@
[{'Z': 'Affaire'}])],
None, None, [self.system], {}, [])],
{'x': 999999})
-
+
def test_nonregr9(self):
repo._type_source_cache[999999] = ('Note', 'cards', 999999)
repo._type_source_cache[999998] = ('Note', 'cards', 999998)
@@ -1853,7 +1853,7 @@
[])
],
{'x': 999999})
-
+
def test_nonregr11(self):
repo._type_source_cache[999999] = ('Bookmark', 'system', 999999)
self._test('SET X bookmarked_by Y WHERE X eid %(x)s, Y login "hop"',
@@ -1867,7 +1867,7 @@
[])]
)],
{'x': 999999})
-
+
def test_nonregr12(self):
repo._type_source_cache[999999] = ('Note', 'cards', 999999)
self._test('Any X ORDERBY Z DESC WHERE X modification_date Z, E eid %(x)s, E see_also X',
@@ -1916,7 +1916,7 @@
{'U': 'table1.C0', 'UL': 'table1.C1'},
[])],
{'x': self.session.user.eid})
-
+
def test_nonregr13_2(self):
# identity *not* wrapped into exists.
#
@@ -1954,11 +1954,11 @@
class MSPlannerTwoSameExternalSourcesTC(BasePlannerTC):
"""test planner related feature on a 3-sources repository:
-
+
* 2 rql sources supporting Card
"""
repo = repo
-
+
def setUp(self):
self.o = repo.querier
self.session = repo._sessions.values()[0]
@@ -1982,7 +1982,7 @@
assert 'multisource_crossed_rel' in repo.sources_by_uri['cards'].cross_relations
clear_ms_caches(repo)
_test = test_plan
-
+
def tearDown(self):
undo_monkey_patch()
del self.sources[-1]
@@ -2090,6 +2090,16 @@
)]
)
+ def test_nonregr_dont_cross_rel_source_filtering(self):
+ self.repo._type_source_cache[999999] = ('Note', 'cards', 999999)
+ self._test('Any X,AA,AB WHERE E eid %(x)s, E in_state X, X name AA, X modification_date AB',
+ [('OneFetchStep', [('Any X,AA,AB WHERE 999999 in_state X, X name AA, X modification_date AB, X is State',
+ [{'AA': 'String', 'AB': 'Datetime', 'X': 'State'}])],
+ None, None,
+ [self.rql], {}, []
+ )],
+ {'x': 999999})
+
if __name__ == '__main__':
from logilab.common.testlib import unittest_main