|
1 # copyright 2003-2014 LOGILAB S.A. (Paris, FRANCE), all rights reserved. |
|
2 # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr |
|
3 # |
|
4 # This file is part of CubicWeb. |
|
5 # |
|
6 # CubicWeb is free software: you can redistribute it and/or modify it under the |
|
7 # terms of the GNU Lesser General Public License as published by the Free |
|
8 # Software Foundation, either version 2.1 of the License, or (at your option) |
|
9 # any later version. |
|
10 # |
|
11 # CubicWeb is distributed in the hope that it will be useful, but WITHOUT |
|
12 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS |
|
13 # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more |
|
14 # details. |
|
15 # |
|
16 # You should have received a copy of the GNU Lesser General Public License along |
|
17 # with CubicWeb. If not, see <http://www.gnu.org/licenses/>. |
|
18 """Helper classes to execute RQL queries on a set of sources, performing |
|
19 security checking and data aggregation. |
|
20 """ |
|
21 from __future__ import print_function |
|
22 |
|
23 __docformat__ = "restructuredtext en" |
|
24 |
|
25 from itertools import repeat |
|
26 |
|
27 from six import text_type, string_types, integer_types |
|
28 from six.moves import range |
|
29 |
|
30 from rql import RQLSyntaxError, CoercionError |
|
31 from rql.stmts import Union |
|
32 from rql.nodes import ETYPE_PYOBJ_MAP, etype_from_pyobj, Relation, Exists, Not |
|
33 from yams import BASE_TYPES |
|
34 |
|
35 from cubicweb import ValidationError, Unauthorized, UnknownEid |
|
36 from cubicweb.rqlrewrite import RQLRelationRewriter |
|
37 from cubicweb import Binary, server |
|
38 from cubicweb.rset import ResultSet |
|
39 |
|
40 from cubicweb.utils import QueryCache, RepeatList |
|
41 from cubicweb.server.rqlannotation import SQLGenAnnotator, set_qdata |
|
42 from cubicweb.server.ssplanner import READ_ONLY_RTYPES, add_types_restriction |
|
43 from cubicweb.server.edition import EditedEntity |
|
44 from cubicweb.server.ssplanner import SSPlanner |
|
45 from cubicweb.statsd_logger import statsd_timeit, statsd_c |
|
46 |
|
47 ETYPE_PYOBJ_MAP[Binary] = 'Bytes' |
|
48 |
|
49 |
|
50 def empty_rset(rql, args, rqlst=None): |
|
51 """build an empty result set object""" |
|
52 return ResultSet([], rql, args, rqlst=rqlst) |
|
53 |
|
54 def update_varmap(varmap, selected, table): |
|
55 """return a sql schema to store RQL query result""" |
|
56 for i, term in enumerate(selected): |
|
57 key = term.as_string() |
|
58 value = '%s.C%s' % (table, i) |
|
59 if varmap.get(key, value) != value: |
|
60 raise Exception('variable name conflict on %s: got %s / %s' |
|
61 % (key, value, varmap)) |
|
62 varmap[key] = value |
|
63 |
|
64 # permission utilities ######################################################## |
|
65 |
|
66 def check_no_password_selected(rqlst): |
|
67 """check that Password entities are not selected""" |
|
68 for solution in rqlst.solutions: |
|
69 for var, etype in solution.items(): |
|
70 if etype == 'Password': |
|
71 raise Unauthorized('Password selection is not allowed (%s)' % var) |
|
72 |
|
73 def term_etype(cnx, term, solution, args): |
|
74 """return the entity type for the given term (a VariableRef or a Constant |
|
75 node) |
|
76 """ |
|
77 try: |
|
78 return solution[term.name] |
|
79 except AttributeError: |
|
80 return cnx.entity_metas(term.eval(args))['type'] |
|
81 |
|
82 def check_relations_read_access(cnx, select, args): |
|
83 """Raise :exc:`Unauthorized` if the given user doesn't have credentials to |
|
84 read relations used in the given syntax tree |
|
85 """ |
|
86 # use `term_etype` since we've to deal with rewritten constants here, |
|
87 # when used as an external source by another repository. |
|
88 # XXX what about local read security w/ those rewritten constants... |
|
89 # XXX constants can also happen in some queries generated by req.find() |
|
90 DBG = (server.DEBUG & server.DBG_SEC) and 'read' in server._SECURITY_CAPS |
|
91 schema = cnx.repo.schema |
|
92 user = cnx.user |
|
93 if select.where is not None: |
|
94 for rel in select.where.iget_nodes(Relation): |
|
95 for solution in select.solutions: |
|
96 # XXX has_text may have specific perm ? |
|
97 if rel.r_type in READ_ONLY_RTYPES: |
|
98 continue |
|
99 rschema = schema.rschema(rel.r_type) |
|
100 if rschema.final: |
|
101 eschema = schema.eschema(term_etype(cnx, rel.children[0], |
|
102 solution, args)) |
|
103 rdef = eschema.rdef(rschema) |
|
104 else: |
|
105 rdef = rschema.rdef(term_etype(cnx, rel.children[0], |
|
106 solution, args), |
|
107 term_etype(cnx, rel.children[1].children[0], |
|
108 solution, args)) |
|
109 if not user.matching_groups(rdef.get_groups('read')): |
|
110 if DBG: |
|
111 print('check_read_access: %s %s does not match %s' % |
|
112 (rdef, user.groups, rdef.get_groups('read'))) |
|
113 # XXX rqlexpr not allowed |
|
114 raise Unauthorized('read', rel.r_type) |
|
115 if DBG: |
|
116 print('check_read_access: %s %s matches %s' % |
|
117 (rdef, user.groups, rdef.get_groups('read'))) |
|
118 |
|
119 def get_local_checks(cnx, rqlst, solution): |
|
120 """Check that the given user has credentials to access data read by the |
|
121 query and return a dict defining necessary "local checks" (i.e. rql |
|
122 expression in read permission defined in the schema) where no group grants |
|
123 him the permission. |
|
124 |
|
125 Returned dictionary's keys are variable names and values the rql expressions |
|
126 for this variable (with the given solution). |
|
127 |
|
128 Raise :exc:`Unauthorized` if access is known to be defined, i.e. if there is |
|
129 no matching group and no local permissions. |
|
130 """ |
|
131 DBG = (server.DEBUG & server.DBG_SEC) and 'read' in server._SECURITY_CAPS |
|
132 schema = cnx.repo.schema |
|
133 user = cnx.user |
|
134 localchecks = {} |
|
135 # iterate on defined_vars and not on solutions to ignore column aliases |
|
136 for varname in rqlst.defined_vars: |
|
137 eschema = schema.eschema(solution[varname]) |
|
138 if eschema.final: |
|
139 continue |
|
140 if not user.matching_groups(eschema.get_groups('read')): |
|
141 erqlexprs = eschema.get_rqlexprs('read') |
|
142 if not erqlexprs: |
|
143 ex = Unauthorized('read', solution[varname]) |
|
144 ex.var = varname |
|
145 if DBG: |
|
146 print('check_read_access: %s %s %s %s' % |
|
147 (varname, eschema, user.groups, eschema.get_groups('read'))) |
|
148 raise ex |
|
149 # don't insert security on variable only referenced by 'NOT X relation Y' or |
|
150 # 'NOT EXISTS(X relation Y)' |
|
151 varinfo = rqlst.defined_vars[varname].stinfo |
|
152 if varinfo['selected'] or ( |
|
153 len([r for r in varinfo['relations'] |
|
154 if (not schema.rschema(r.r_type).final |
|
155 and ((isinstance(r.parent, Exists) and r.parent.neged(strict=True)) |
|
156 or isinstance(r.parent, Not)))]) |
|
157 != |
|
158 len(varinfo['relations'])): |
|
159 localchecks[varname] = erqlexprs |
|
160 return localchecks |
|
161 |
|
162 |
|
163 # Plans ####################################################################### |
|
164 |
|
165 class ExecutionPlan(object): |
|
166 """the execution model of a rql query, composed of querier steps""" |
|
167 |
|
168 def __init__(self, querier, rqlst, args, cnx): |
|
169 # original rql syntax tree |
|
170 self.rqlst = rqlst |
|
171 self.args = args or {} |
|
172 # cnx executing the query |
|
173 self.cnx = cnx |
|
174 # quick reference to the system source |
|
175 self.syssource = cnx.repo.system_source |
|
176 # execution steps |
|
177 self.steps = [] |
|
178 # various resource accesors |
|
179 self.querier = querier |
|
180 self.schema = querier.schema |
|
181 self.sqlannotate = querier.sqlgen_annotate |
|
182 self.rqlhelper = cnx.vreg.rqlhelper |
|
183 |
|
184 def annotate_rqlst(self): |
|
185 if not self.rqlst.annotated: |
|
186 self.rqlhelper.annotate(self.rqlst) |
|
187 |
|
188 def add_step(self, step): |
|
189 """add a step to the plan""" |
|
190 self.steps.append(step) |
|
191 |
|
192 def sqlexec(self, sql, args=None): |
|
193 return self.syssource.sqlexec(self.cnx, sql, args) |
|
194 |
|
195 def execute(self): |
|
196 """execute a plan and return resulting rows""" |
|
197 for step in self.steps: |
|
198 result = step.execute() |
|
199 # the latest executed step contains the full query result |
|
200 return result |
|
201 |
|
202 def preprocess(self, union, security=True): |
|
203 """insert security when necessary then annotate rql st for sql generation |
|
204 |
|
205 return rqlst to actually execute |
|
206 """ |
|
207 cached = None |
|
208 if security and self.cnx.read_security: |
|
209 # ensure security is turned of when security is inserted, |
|
210 # else we may loop for ever... |
|
211 if self.cnx.transaction_data.get('security-rqlst-cache'): |
|
212 key = self.cache_key |
|
213 else: |
|
214 key = None |
|
215 if key is not None and key in self.cnx.transaction_data: |
|
216 cachedunion, args = self.cnx.transaction_data[key] |
|
217 union.children[:] = [] |
|
218 for select in cachedunion.children: |
|
219 union.append(select) |
|
220 union.has_text_query = cachedunion.has_text_query |
|
221 args.update(self.args) |
|
222 self.args = args |
|
223 cached = True |
|
224 else: |
|
225 with self.cnx.security_enabled(read=False): |
|
226 noinvariant = self._insert_security(union) |
|
227 if key is not None: |
|
228 self.cnx.transaction_data[key] = (union, self.args) |
|
229 else: |
|
230 noinvariant = () |
|
231 if cached is None: |
|
232 self.rqlhelper.simplify(union) |
|
233 self.sqlannotate(union) |
|
234 set_qdata(self.schema.rschema, union, noinvariant) |
|
235 if union.has_text_query: |
|
236 self.cache_key = None |
|
237 |
|
238 def _insert_security(self, union): |
|
239 noinvariant = set() |
|
240 for select in union.children[:]: |
|
241 for subquery in select.with_: |
|
242 self._insert_security(subquery.query) |
|
243 localchecks, restricted = self._check_permissions(select) |
|
244 if any(localchecks): |
|
245 self.cnx.rql_rewriter.insert_local_checks( |
|
246 select, self.args, localchecks, restricted, noinvariant) |
|
247 return noinvariant |
|
248 |
|
249 def _check_permissions(self, rqlst): |
|
250 """Return a dict defining "local checks", i.e. RQLExpression defined in |
|
251 the schema that should be inserted in the original query, together with |
|
252 a set of variable names which requires some security to be inserted. |
|
253 |
|
254 Solutions where a variable has a type which the user can't definitly |
|
255 read are removed, else if the user *may* read it (i.e. if an rql |
|
256 expression is defined for the "read" permission of the related type), |
|
257 the local checks dict is updated. |
|
258 |
|
259 The local checks dict has entries for each different local check |
|
260 necessary, with associated solutions as value, a local check being |
|
261 defined by a list of 2-uple (variable name, rql expressions) for each |
|
262 variable which has to be checked. Solutions which don't require local |
|
263 checks will be associated to the empty tuple key. |
|
264 |
|
265 Note rqlst should not have been simplified at this point. |
|
266 """ |
|
267 cnx = self.cnx |
|
268 msgs = [] |
|
269 # dict(varname: eid), allowing to check rql expression for variables |
|
270 # which have a known eid |
|
271 varkwargs = {} |
|
272 if not cnx.transaction_data.get('security-rqlst-cache'): |
|
273 for var in rqlst.defined_vars.values(): |
|
274 if var.stinfo['constnode'] is not None: |
|
275 eid = var.stinfo['constnode'].eval(self.args) |
|
276 varkwargs[var.name] = int(eid) |
|
277 # dictionary of variables restricted for security reason |
|
278 localchecks = {} |
|
279 restricted_vars = set() |
|
280 newsolutions = [] |
|
281 for solution in rqlst.solutions: |
|
282 try: |
|
283 localcheck = get_local_checks(cnx, rqlst, solution) |
|
284 except Unauthorized as ex: |
|
285 msg = 'remove %s from solutions since %s has no %s access to %s' |
|
286 msg %= (solution, cnx.user.login, ex.args[0], ex.args[1]) |
|
287 msgs.append(msg) |
|
288 LOGGER.info(msg) |
|
289 else: |
|
290 newsolutions.append(solution) |
|
291 # try to benefit of rqlexpr.check cache for entities which |
|
292 # are specified by eid in query'args |
|
293 for varname, eid in varkwargs.items(): |
|
294 try: |
|
295 rqlexprs = localcheck.pop(varname) |
|
296 except KeyError: |
|
297 continue |
|
298 # if entity has been added in the current transaction, the |
|
299 # user can read it whatever rql expressions are associated |
|
300 # to its type |
|
301 if cnx.added_in_transaction(eid): |
|
302 continue |
|
303 for rqlexpr in rqlexprs: |
|
304 if rqlexpr.check(cnx, eid): |
|
305 break |
|
306 else: |
|
307 raise Unauthorized('No read acces on %r with eid %i.' % (var, eid)) |
|
308 # mark variables protected by an rql expression |
|
309 restricted_vars.update(localcheck) |
|
310 # turn local check into a dict key |
|
311 localcheck = tuple(sorted(localcheck.items())) |
|
312 localchecks.setdefault(localcheck, []).append(solution) |
|
313 # raise Unautorized exception if the user can't access to any solution |
|
314 if not newsolutions: |
|
315 raise Unauthorized('\n'.join(msgs)) |
|
316 # if there is some message, solutions have been modified and must be |
|
317 # reconsidered by the syntax treee |
|
318 if msgs: |
|
319 rqlst.set_possible_types(newsolutions) |
|
320 return localchecks, restricted_vars |
|
321 |
|
322 def finalize(self, select, solutions, insertedvars): |
|
323 rqlst = Union() |
|
324 rqlst.append(select) |
|
325 for mainvarname, rschema, newvarname in insertedvars: |
|
326 nvartype = str(rschema.objects(solutions[0][mainvarname])[0]) |
|
327 for sol in solutions: |
|
328 sol[newvarname] = nvartype |
|
329 select.clean_solutions(solutions) |
|
330 add_types_restriction(self.schema, select) |
|
331 self.rqlhelper.annotate(rqlst) |
|
332 self.preprocess(rqlst, security=False) |
|
333 return rqlst |
|
334 |
|
335 |
|
336 class InsertPlan(ExecutionPlan): |
|
337 """an execution model specific to the INSERT rql query |
|
338 """ |
|
339 |
|
340 def __init__(self, querier, rqlst, args, cnx): |
|
341 ExecutionPlan.__init__(self, querier, rqlst, args, cnx) |
|
342 # save originally selected variable, we may modify this |
|
343 # dictionary for substitution (query parameters) |
|
344 self.selected = rqlst.selection |
|
345 # list of rows of entities definition (ssplanner.EditedEntity) |
|
346 self.e_defs = [[]] |
|
347 # list of new relation definition (3-uple (from_eid, r_type, to_eid) |
|
348 self.r_defs = set() |
|
349 # indexes to track entity definitions bound to relation definitions |
|
350 self._r_subj_index = {} |
|
351 self._r_obj_index = {} |
|
352 self._expanded_r_defs = {} |
|
353 |
|
354 def add_entity_def(self, edef): |
|
355 """add an entity definition to build""" |
|
356 self.e_defs[-1].append(edef) |
|
357 |
|
358 def add_relation_def(self, rdef): |
|
359 """add an relation definition to build""" |
|
360 self.r_defs.add(rdef) |
|
361 if not isinstance(rdef[0], int): |
|
362 self._r_subj_index.setdefault(rdef[0], []).append(rdef) |
|
363 if not isinstance(rdef[2], int): |
|
364 self._r_obj_index.setdefault(rdef[2], []).append(rdef) |
|
365 |
|
366 def substitute_entity_def(self, edef, edefs): |
|
367 """substitute an incomplete entity definition by a list of complete |
|
368 equivalents |
|
369 |
|
370 e.g. on queries such as :: |
|
371 INSERT Personne X, Societe Y: X nom N, Y nom 'toto', X travaille Y |
|
372 WHERE U login 'admin', U login N |
|
373 |
|
374 X will be inserted as many times as U exists, and so the X travaille Y |
|
375 relations as to be added as many time as X is inserted |
|
376 """ |
|
377 if not edefs or not self.e_defs: |
|
378 # no result, no entity will be created |
|
379 self.e_defs = () |
|
380 return |
|
381 # first remove the incomplete entity definition |
|
382 colidx = self.e_defs[0].index(edef) |
|
383 for i, row in enumerate(self.e_defs[:]): |
|
384 self.e_defs[i][colidx] = edefs[0] |
|
385 samplerow = self.e_defs[i] |
|
386 for edef_ in edefs[1:]: |
|
387 row = [ed.clone() for i, ed in enumerate(samplerow) |
|
388 if i != colidx] |
|
389 row.insert(colidx, edef_) |
|
390 self.e_defs.append(row) |
|
391 # now, see if this entity def is referenced as subject in some relation |
|
392 # definition |
|
393 if edef in self._r_subj_index: |
|
394 for rdef in self._r_subj_index[edef]: |
|
395 expanded = self._expanded(rdef) |
|
396 result = [] |
|
397 for exp_rdef in expanded: |
|
398 for edef_ in edefs: |
|
399 result.append( (edef_, exp_rdef[1], exp_rdef[2]) ) |
|
400 self._expanded_r_defs[rdef] = result |
|
401 # and finally, see if this entity def is referenced as object in some |
|
402 # relation definition |
|
403 if edef in self._r_obj_index: |
|
404 for rdef in self._r_obj_index[edef]: |
|
405 expanded = self._expanded(rdef) |
|
406 result = [] |
|
407 for exp_rdef in expanded: |
|
408 for edef_ in edefs: |
|
409 result.append( (exp_rdef[0], exp_rdef[1], edef_) ) |
|
410 self._expanded_r_defs[rdef] = result |
|
411 |
|
412 def _expanded(self, rdef): |
|
413 """return expanded value for the given relation definition""" |
|
414 try: |
|
415 return self._expanded_r_defs[rdef] |
|
416 except KeyError: |
|
417 self.r_defs.remove(rdef) |
|
418 return [rdef] |
|
419 |
|
420 def relation_defs(self): |
|
421 """return the list for relation definitions to insert""" |
|
422 for rdefs in self._expanded_r_defs.values(): |
|
423 for rdef in rdefs: |
|
424 yield rdef |
|
425 for rdef in self.r_defs: |
|
426 yield rdef |
|
427 |
|
428 def insert_entity_defs(self): |
|
429 """return eids of inserted entities in a suitable form for the resulting |
|
430 result set, e.g.: |
|
431 |
|
432 e.g. on queries such as :: |
|
433 INSERT Personne X, Societe Y: X nom N, Y nom 'toto', X travaille Y |
|
434 WHERE U login 'admin', U login N |
|
435 |
|
436 if there is two entities matching U, the result set will look like |
|
437 [(eidX1, eidY1), (eidX2, eidY2)] |
|
438 """ |
|
439 cnx = self.cnx |
|
440 repo = cnx.repo |
|
441 results = [] |
|
442 for row in self.e_defs: |
|
443 results.append([repo.glob_add_entity(cnx, edef) |
|
444 for edef in row]) |
|
445 return results |
|
446 |
|
447 def insert_relation_defs(self): |
|
448 cnx = self.cnx |
|
449 repo = cnx.repo |
|
450 edited_entities = {} |
|
451 relations = {} |
|
452 for subj, rtype, obj in self.relation_defs(): |
|
453 # if a string is given into args instead of an int, we get it here |
|
454 if isinstance(subj, string_types): |
|
455 subj = int(subj) |
|
456 elif not isinstance(subj, integer_types): |
|
457 subj = subj.entity.eid |
|
458 if isinstance(obj, string_types): |
|
459 obj = int(obj) |
|
460 elif not isinstance(obj, integer_types): |
|
461 obj = obj.entity.eid |
|
462 if repo.schema.rschema(rtype).inlined: |
|
463 if subj not in edited_entities: |
|
464 entity = cnx.entity_from_eid(subj) |
|
465 edited = EditedEntity(entity) |
|
466 edited_entities[subj] = edited |
|
467 else: |
|
468 edited = edited_entities[subj] |
|
469 edited.edited_attribute(rtype, obj) |
|
470 else: |
|
471 if rtype in relations: |
|
472 relations[rtype].append((subj, obj)) |
|
473 else: |
|
474 relations[rtype] = [(subj, obj)] |
|
475 repo.glob_add_relations(cnx, relations) |
|
476 for edited in edited_entities.values(): |
|
477 repo.glob_update_entity(cnx, edited) |
|
478 |
|
479 |
|
480 class QuerierHelper(object): |
|
481 """helper class to execute rql queries, putting all things together""" |
|
482 |
|
483 def __init__(self, repo, schema): |
|
484 # system info helper |
|
485 self._repo = repo |
|
486 # instance schema |
|
487 self.set_schema(schema) |
|
488 |
|
489 def set_schema(self, schema): |
|
490 self.schema = schema |
|
491 repo = self._repo |
|
492 # rql st and solution cache. |
|
493 self._rql_cache = QueryCache(repo.config['rql-cache-size']) |
|
494 # rql cache key cache. Don't bother using a Cache instance: we should |
|
495 # have a limited number of queries in there, since there are no entries |
|
496 # in this cache for user queries (which have no args) |
|
497 self._rql_ck_cache = {} |
|
498 # some cache usage stats |
|
499 self.cache_hit, self.cache_miss = 0, 0 |
|
500 # rql parsing / analysing helper |
|
501 self.solutions = repo.vreg.solutions |
|
502 rqlhelper = repo.vreg.rqlhelper |
|
503 # set backend on the rql helper, will be used for function checking |
|
504 rqlhelper.backend = repo.config.system_source_config['db-driver'] |
|
505 self._parse = rqlhelper.parse |
|
506 self._annotate = rqlhelper.annotate |
|
507 # rql planner |
|
508 self._planner = SSPlanner(schema, rqlhelper) |
|
509 # sql generation annotator |
|
510 self.sqlgen_annotate = SQLGenAnnotator(schema).annotate |
|
511 |
|
512 def parse(self, rql, annotate=False): |
|
513 """return a rql syntax tree for the given rql""" |
|
514 try: |
|
515 return self._parse(text_type(rql), annotate=annotate) |
|
516 except UnicodeError: |
|
517 raise RQLSyntaxError(rql) |
|
518 |
|
519 def plan_factory(self, rqlst, args, cnx): |
|
520 """create an execution plan for an INSERT RQL query""" |
|
521 if rqlst.TYPE == 'insert': |
|
522 return InsertPlan(self, rqlst, args, cnx) |
|
523 return ExecutionPlan(self, rqlst, args, cnx) |
|
524 |
|
525 @statsd_timeit |
|
526 def execute(self, cnx, rql, args=None, build_descr=True): |
|
527 """execute a rql query, return resulting rows and their description in |
|
528 a `ResultSet` object |
|
529 |
|
530 * `rql` should be a Unicode string or a plain ASCII string |
|
531 * `args` the optional parameters dictionary associated to the query |
|
532 * `build_descr` is a boolean flag indicating if the description should |
|
533 be built on select queries (if false, the description will be en empty |
|
534 list) |
|
535 |
|
536 on INSERT queries, there will be one row with the eid of each inserted |
|
537 entity |
|
538 |
|
539 result for DELETE and SET queries is undefined yet |
|
540 |
|
541 to maximize the rql parsing/analyzing cache performance, you should |
|
542 always use substitute arguments in queries (i.e. avoid query such as |
|
543 'Any X WHERE X eid 123'!) |
|
544 """ |
|
545 if server.DEBUG & (server.DBG_RQL | server.DBG_SQL): |
|
546 if server.DEBUG & (server.DBG_MORE | server.DBG_SQL): |
|
547 print('*'*80) |
|
548 print('querier input', repr(rql), repr(args)) |
|
549 # parse the query and binds variables |
|
550 cachekey = (rql,) |
|
551 try: |
|
552 if args: |
|
553 # search for named args in query which are eids (hence |
|
554 # influencing query's solutions) |
|
555 eidkeys = self._rql_ck_cache[rql] |
|
556 if eidkeys: |
|
557 # if there are some, we need a better cache key, eg (rql + |
|
558 # entity type of each eid) |
|
559 try: |
|
560 cachekey = self._repo.querier_cache_key(cnx, rql, |
|
561 args, eidkeys) |
|
562 except UnknownEid: |
|
563 # we want queries such as "Any X WHERE X eid 9999" |
|
564 # return an empty result instead of raising UnknownEid |
|
565 return empty_rset(rql, args) |
|
566 rqlst = self._rql_cache[cachekey] |
|
567 self.cache_hit += 1 |
|
568 statsd_c('cache_hit') |
|
569 except KeyError: |
|
570 self.cache_miss += 1 |
|
571 statsd_c('cache_miss') |
|
572 rqlst = self.parse(rql) |
|
573 try: |
|
574 # compute solutions for rqlst and return named args in query |
|
575 # which are eids. Notice that if you may not need `eidkeys`, we |
|
576 # have to compute solutions anyway (kept as annotation on the |
|
577 # tree) |
|
578 eidkeys = self.solutions(cnx, rqlst, args) |
|
579 except UnknownEid: |
|
580 # we want queries such as "Any X WHERE X eid 9999" return an |
|
581 # empty result instead of raising UnknownEid |
|
582 return empty_rset(rql, args) |
|
583 if args and rql not in self._rql_ck_cache: |
|
584 self._rql_ck_cache[rql] = eidkeys |
|
585 if eidkeys: |
|
586 cachekey = self._repo.querier_cache_key(cnx, rql, args, |
|
587 eidkeys) |
|
588 self._rql_cache[cachekey] = rqlst |
|
589 if rqlst.TYPE != 'select': |
|
590 if cnx.read_security: |
|
591 check_no_password_selected(rqlst) |
|
592 cachekey = None |
|
593 else: |
|
594 if cnx.read_security: |
|
595 for select in rqlst.children: |
|
596 check_no_password_selected(select) |
|
597 check_relations_read_access(cnx, select, args) |
|
598 # on select query, always copy the cached rqlst so we don't have to |
|
599 # bother modifying it. This is not necessary on write queries since |
|
600 # a new syntax tree is built from them. |
|
601 rqlst = rqlst.copy() |
|
602 # Rewrite computed relations |
|
603 rewriter = RQLRelationRewriter(cnx) |
|
604 rewriter.rewrite(rqlst, args) |
|
605 self._annotate(rqlst) |
|
606 if args: |
|
607 # different SQL generated when some argument is None or not (IS |
|
608 # NULL). This should be considered when computing sql cache key |
|
609 cachekey += tuple(sorted([k for k, v in args.items() |
|
610 if v is None])) |
|
611 # make an execution plan |
|
612 plan = self.plan_factory(rqlst, args, cnx) |
|
613 plan.cache_key = cachekey |
|
614 self._planner.build_plan(plan) |
|
615 # execute the plan |
|
616 try: |
|
617 results = plan.execute() |
|
618 except (Unauthorized, ValidationError): |
|
619 # getting an Unauthorized/ValidationError exception means the |
|
620 # transaction must be rolled back |
|
621 # |
|
622 # notes: |
|
623 # * we should not reset the connections set here, since we don't want the |
|
624 # connection to loose it during processing |
|
625 # * don't rollback if we're in the commit process, will be handled |
|
626 # by the connection |
|
627 if cnx.commit_state is None: |
|
628 cnx.commit_state = 'uncommitable' |
|
629 raise |
|
630 # build a description for the results if necessary |
|
631 descr = () |
|
632 if build_descr: |
|
633 if rqlst.TYPE == 'select': |
|
634 # sample selection |
|
635 if len(rqlst.children) == 1 and len(rqlst.children[0].solutions) == 1: |
|
636 # easy, all lines are identical |
|
637 selected = rqlst.children[0].selection |
|
638 solution = rqlst.children[0].solutions[0] |
|
639 description = _make_description(selected, args, solution) |
|
640 descr = RepeatList(len(results), tuple(description)) |
|
641 else: |
|
642 # hard, delegate the work :o) |
|
643 descr = manual_build_descr(cnx, rqlst, args, results) |
|
644 elif rqlst.TYPE == 'insert': |
|
645 # on insert plan, some entities may have been auto-casted, |
|
646 # so compute description manually even if there is only |
|
647 # one solution |
|
648 basedescr = [None] * len(plan.selected) |
|
649 todetermine = list(zip(range(len(plan.selected)), repeat(False))) |
|
650 descr = _build_descr(cnx, results, basedescr, todetermine) |
|
651 # FIXME: get number of affected entities / relations on non |
|
652 # selection queries ? |
|
653 # return a result set object |
|
654 return ResultSet(results, rql, args, descr) |
|
655 |
|
656 # these are overridden by set_log_methods below |
|
657 # only defining here to prevent pylint from complaining |
|
658 info = warning = error = critical = exception = debug = lambda msg,*a,**kw: None |
|
659 |
|
660 from logging import getLogger |
|
661 from cubicweb import set_log_methods |
|
662 LOGGER = getLogger('cubicweb.querier') |
|
663 set_log_methods(QuerierHelper, LOGGER) |
|
664 |
|
665 |
|
666 def manual_build_descr(cnx, rqlst, args, result): |
|
667 """build a description for a given result by analysing each row |
|
668 |
|
669 XXX could probably be done more efficiently during execution of query |
|
670 """ |
|
671 # not so easy, looks for variable which changes from one solution |
|
672 # to another |
|
673 unstables = rqlst.get_variable_indices() |
|
674 basedescr = [] |
|
675 todetermine = [] |
|
676 for i in range(len(rqlst.children[0].selection)): |
|
677 ttype = _selection_idx_type(i, rqlst, args) |
|
678 if ttype is None or ttype == 'Any': |
|
679 ttype = None |
|
680 isfinal = True |
|
681 else: |
|
682 isfinal = ttype in BASE_TYPES |
|
683 if ttype is None or i in unstables: |
|
684 basedescr.append(None) |
|
685 todetermine.append( (i, isfinal) ) |
|
686 else: |
|
687 basedescr.append(ttype) |
|
688 if not todetermine: |
|
689 return RepeatList(len(result), tuple(basedescr)) |
|
690 return _build_descr(cnx, result, basedescr, todetermine) |
|
691 |
|
692 def _build_descr(cnx, result, basedescription, todetermine): |
|
693 description = [] |
|
694 entity_metas = cnx.entity_metas |
|
695 todel = [] |
|
696 for i, row in enumerate(result): |
|
697 row_descr = basedescription[:] |
|
698 for index, isfinal in todetermine: |
|
699 value = row[index] |
|
700 if value is None: |
|
701 # None value inserted by an outer join, no type |
|
702 row_descr[index] = None |
|
703 continue |
|
704 if isfinal: |
|
705 row_descr[index] = etype_from_pyobj(value) |
|
706 else: |
|
707 try: |
|
708 row_descr[index] = entity_metas(value)['type'] |
|
709 except UnknownEid: |
|
710 cnx.error('wrong eid %s in repository, you should ' |
|
711 'db-check the database' % value) |
|
712 todel.append(i) |
|
713 break |
|
714 else: |
|
715 description.append(tuple(row_descr)) |
|
716 for i in reversed(todel): |
|
717 del result[i] |
|
718 return description |
|
719 |
|
720 def _make_description(selected, args, solution): |
|
721 """return a description for a result set""" |
|
722 description = [] |
|
723 for term in selected: |
|
724 description.append(term.get_type(solution, args)) |
|
725 return description |
|
726 |
|
727 def _selection_idx_type(i, rqlst, args): |
|
728 """try to return type of term at index `i` of the rqlst's selection""" |
|
729 for select in rqlst.children: |
|
730 term = select.selection[i] |
|
731 for solution in select.solutions: |
|
732 try: |
|
733 ttype = term.get_type(solution, args) |
|
734 if ttype is not None: |
|
735 return ttype |
|
736 except CoercionError: |
|
737 return None |