author | Sylvain Thénault <sylvain.thenault@logilab.fr> |
Wed, 03 Nov 2010 17:17:56 +0100 | |
branch | stable |
changeset 6668 | d651bf5a66a9 |
parent 6655 | 75112ff0511d |
child 6672 | 2008fd2f628c |
permissions | -rw-r--r-- |
5421
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
1 |
# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
2 |
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
3 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
4 |
# This file is part of CubicWeb. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
5 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
6 |
# CubicWeb is free software: you can redistribute it and/or modify it under the |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
7 |
# terms of the GNU Lesser General Public License as published by the Free |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
8 |
# Software Foundation, either version 2.1 of the License, or (at your option) |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
9 |
# any later version. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
10 |
# |
5424
8ecbcbff9777
replace logilab-common by CubicWeb in disclaimer
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5421
diff
changeset
|
11 |
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT |
5421
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
12 |
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
13 |
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
14 |
# details. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
15 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
16 |
# You should have received a copy of the GNU Lesser General Public License along |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
17 |
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>. |
6128 | 18 |
"""Source to query another RQL repository using pyro""" |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
19 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
20 |
__docformat__ = "restructuredtext en" |
6582
8eb7883b4223
[pylint] fix a bug of pylint detected errors and i18n pb (calling builtins._ instead of req._)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6427
diff
changeset
|
21 |
_ = unicode |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
22 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
23 |
import threading |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
24 |
from os.path import join |
1016
26387b836099
use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
25 |
from time import mktime |
26387b836099
use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
26 |
from datetime import datetime |
1952
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1805
diff
changeset
|
27 |
from base64 import b64decode |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
28 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
29 |
from Pyro.errors import PyroError, ConnectionClosedError |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
30 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
31 |
from logilab.common.configuration import REQUIRED |
6625
54847e2c4a50
[pyro source] new option to control wether external entities should be skipped or not
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6582
diff
changeset
|
32 |
from logilab.common.optik_ext import check_yn |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
33 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
34 |
from rql.nodes import Constant |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
35 |
from rql.utils import rqlvar_maker |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
36 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
37 |
from cubicweb import dbapi, server |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
38 |
from cubicweb import BadConnectionId, UnknownEid, ConnectionError |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
39 |
from cubicweb.cwconfig import register_persistent_options |
2625
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2593
diff
changeset
|
40 |
from cubicweb.server.sources import (AbstractSource, ConnectionWrapper, |
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2593
diff
changeset
|
41 |
TimedCache, dbg_st_search, dbg_results) |
5582
3e133b29a1a4
[rql2sql] follow rql 0.26.1 changes: NOT nodes normalization, allowing simplification of sql generation, and fix #XXX
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5426
diff
changeset
|
42 |
from cubicweb.server.msplanner import neged_relation |
3016
5787d1cc8106
[rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2763
diff
changeset
|
43 |
|
5787d1cc8106
[rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2763
diff
changeset
|
44 |
def uidtype(union, col, etype, args): |
5787d1cc8106
[rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2763
diff
changeset
|
45 |
select, col = union.locate_subquery(col, etype, args) |
5787d1cc8106
[rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2763
diff
changeset
|
46 |
return getattr(select.selection[col], 'uidtype', None) |
5787d1cc8106
[rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2763
diff
changeset
|
47 |
|
6130
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
48 |
def load_mapping_file(mappingfile): |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
49 |
mapping = {} |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
50 |
execfile(mappingfile, mapping) |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
51 |
for junk in ('__builtins__', '__doc__'): |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
52 |
mapping.pop(junk, None) |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
53 |
mapping.setdefault('support_relations', {}) |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
54 |
mapping.setdefault('dont_cross_relations', set()) |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
55 |
mapping.setdefault('cross_relations', set()) |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
56 |
|
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
57 |
# do some basic checks of the mapping content |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
58 |
assert 'support_entities' in mapping, \ |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
59 |
'mapping file should at least define support_entities' |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
60 |
assert isinstance(mapping['support_entities'], dict) |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
61 |
assert isinstance(mapping['support_relations'], dict) |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
62 |
assert isinstance(mapping['dont_cross_relations'], set) |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
63 |
assert isinstance(mapping['cross_relations'], set) |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
64 |
unknown = set(mapping) - set( ('support_entities', 'support_relations', |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
65 |
'dont_cross_relations', 'cross_relations') ) |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
66 |
assert not unknown, 'unknown mapping attribute(s): %s' % unknown |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
67 |
# relations that are necessarily not crossed |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
68 |
mapping['dont_cross_relations'] |= set(('owned_by', 'created_by')) |
6427
c8a5ac2d1eaa
[schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6309
diff
changeset
|
69 |
for rtype in ('is', 'is_instance_of', 'cw_source'): |
6130
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
70 |
assert rtype not in mapping['dont_cross_relations'], \ |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
71 |
'%s relation should not be in dont_cross_relations' % rtype |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
72 |
assert rtype not in mapping['support_relations'], \ |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
73 |
'%s relation should not be in support_relations' % rtype |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
74 |
return mapping |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
75 |
|
3016
5787d1cc8106
[rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2763
diff
changeset
|
76 |
|
1138
22f634977c95
make pylint happy, fix some bugs on the way
sylvain.thenault@logilab.fr
parents:
1016
diff
changeset
|
77 |
class ReplaceByInOperator(Exception): |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
78 |
def __init__(self, eids): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
79 |
self.eids = eids |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
80 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
81 |
class PyroRQLSource(AbstractSource): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
82 |
"""External repository source, using Pyro connection""" |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
83 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
84 |
# boolean telling if modification hooks should be called when something is |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
85 |
# modified in this source |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
86 |
should_call_hooks = False |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
87 |
# boolean telling if the repository should connect to this source during |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
88 |
# migration |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
89 |
connect_for_migration = False |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
90 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
91 |
options = ( |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
92 |
# XXX pyro-ns host/port |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
93 |
('pyro-ns-id', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
94 |
{'type' : 'string', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
95 |
'default': REQUIRED, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
96 |
'help': 'identifier of the repository in the pyro name server', |
5323
329b4f6d18b4
[config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5174
diff
changeset
|
97 |
'group': 'pyro-source', 'level': 0, |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
98 |
}), |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
99 |
('mapping-file', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
100 |
{'type' : 'string', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
101 |
'default': REQUIRED, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
102 |
'help': 'path to a python file with the schema mapping definition', |
5323
329b4f6d18b4
[config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5174
diff
changeset
|
103 |
'group': 'pyro-source', 'level': 1, |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
104 |
}), |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
105 |
('cubicweb-user', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
106 |
{'type' : 'string', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
107 |
'default': REQUIRED, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
108 |
'help': 'user to use for connection on the distant repository', |
5323
329b4f6d18b4
[config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5174
diff
changeset
|
109 |
'group': 'pyro-source', 'level': 0, |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
110 |
}), |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
111 |
('cubicweb-password', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
112 |
{'type' : 'password', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
113 |
'default': '', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
114 |
'help': 'user to use for connection on the distant repository', |
5323
329b4f6d18b4
[config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5174
diff
changeset
|
115 |
'group': 'pyro-source', 'level': 0, |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
116 |
}), |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
117 |
('base-url', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
118 |
{'type' : 'string', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
119 |
'default': '', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
120 |
'help': 'url of the web site for the distant repository, if you want ' |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
121 |
'to generate external link to entities from this repository', |
5323
329b4f6d18b4
[config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5174
diff
changeset
|
122 |
'group': 'pyro-source', 'level': 1, |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
123 |
}), |
6625
54847e2c4a50
[pyro source] new option to control wether external entities should be skipped or not
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6582
diff
changeset
|
124 |
('skip-external-entities', |
54847e2c4a50
[pyro source] new option to control wether external entities should be skipped or not
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6582
diff
changeset
|
125 |
{'type' : 'yn', |
54847e2c4a50
[pyro source] new option to control wether external entities should be skipped or not
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6582
diff
changeset
|
126 |
'default': False, |
54847e2c4a50
[pyro source] new option to control wether external entities should be skipped or not
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6582
diff
changeset
|
127 |
'help': 'should entities not local to the source be considered or not', |
54847e2c4a50
[pyro source] new option to control wether external entities should be skipped or not
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6582
diff
changeset
|
128 |
'group': 'pyro-source', 'level': 0, |
54847e2c4a50
[pyro source] new option to control wether external entities should be skipped or not
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6582
diff
changeset
|
129 |
}), |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
130 |
('pyro-ns-host', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
131 |
{'type' : 'string', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
132 |
'default': None, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
133 |
'help': 'Pyro name server\'s host. If not set, default to the value \ |
2665
0c6281487f90
[pyro] use lgc.pyro_ext, simplify pyro related options
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2625
diff
changeset
|
134 |
from all_in_one.conf. It may contains port information using <host>:<port> notation.', |
5323
329b4f6d18b4
[config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5174
diff
changeset
|
135 |
'group': 'pyro-source', 'level': 1, |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
136 |
}), |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
137 |
('pyro-ns-group', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
138 |
{'type' : 'string', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
139 |
'default': None, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
140 |
'help': 'Pyro name server\'s group where the repository will be \ |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
141 |
registered. If not set, default to the value from all_in_one.conf.', |
6632 | 142 |
'group': 'pyro-source', 'level': 2, |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
143 |
}), |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
144 |
('synchronization-interval', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
145 |
{'type' : 'int', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
146 |
'default': 5*60, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
147 |
'help': 'interval between synchronization with the external \ |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
148 |
repository (default to 5 minutes).', |
5323
329b4f6d18b4
[config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5174
diff
changeset
|
149 |
'group': 'pyro-source', 'level': 2, |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
150 |
}), |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
151 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
152 |
) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
153 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
154 |
PUBLIC_KEYS = AbstractSource.PUBLIC_KEYS + ('base-url',) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
155 |
_conn = None |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
156 |
|
6427
c8a5ac2d1eaa
[schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6309
diff
changeset
|
157 |
def __init__(self, repo, source_config, *args, **kwargs): |
c8a5ac2d1eaa
[schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6309
diff
changeset
|
158 |
AbstractSource.__init__(self, repo, source_config, *args, **kwargs) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
159 |
mappingfile = source_config['mapping-file'] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
160 |
if not mappingfile[0] == '/': |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
161 |
mappingfile = join(repo.config.apphome, mappingfile) |
6427
c8a5ac2d1eaa
[schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6309
diff
changeset
|
162 |
try: |
c8a5ac2d1eaa
[schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6309
diff
changeset
|
163 |
mapping = load_mapping_file(mappingfile) |
c8a5ac2d1eaa
[schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6309
diff
changeset
|
164 |
except IOError: |
c8a5ac2d1eaa
[schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6309
diff
changeset
|
165 |
self.disabled = True |
c8a5ac2d1eaa
[schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6309
diff
changeset
|
166 |
self.error('cant read mapping file %s, source disabled', |
c8a5ac2d1eaa
[schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6309
diff
changeset
|
167 |
mappingfile) |
c8a5ac2d1eaa
[schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6309
diff
changeset
|
168 |
self.support_entities = {} |
c8a5ac2d1eaa
[schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6309
diff
changeset
|
169 |
self.support_relations = {} |
c8a5ac2d1eaa
[schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6309
diff
changeset
|
170 |
self.dont_cross_relations = set() |
c8a5ac2d1eaa
[schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6309
diff
changeset
|
171 |
self.cross_relations = set() |
c8a5ac2d1eaa
[schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6309
diff
changeset
|
172 |
else: |
c8a5ac2d1eaa
[schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6309
diff
changeset
|
173 |
self.support_entities = mapping['support_entities'] |
c8a5ac2d1eaa
[schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6309
diff
changeset
|
174 |
self.support_relations = mapping['support_relations'] |
c8a5ac2d1eaa
[schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6309
diff
changeset
|
175 |
self.dont_cross_relations = mapping['dont_cross_relations'] |
c8a5ac2d1eaa
[schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6309
diff
changeset
|
176 |
self.cross_relations = mapping['cross_relations'] |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
177 |
baseurl = source_config.get('base-url') |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
178 |
if baseurl and not baseurl.endswith('/'): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
179 |
source_config['base-url'] += '/' |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
180 |
self.config = source_config |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
181 |
myoptions = (('%s.latest-update-time' % self.uri, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
182 |
{'type' : 'int', 'sitewide': True, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
183 |
'default': 0, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
184 |
'help': _('timestamp of the latest source synchronization.'), |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
185 |
'group': 'sources', |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
186 |
}),) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
187 |
register_persistent_options(myoptions) |
5642
6a90357b9769
TimedCache now only accepts values expressed in seconds
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
5426
diff
changeset
|
188 |
self._query_cache = TimedCache(1800) |
6625
54847e2c4a50
[pyro source] new option to control wether external entities should be skipped or not
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6582
diff
changeset
|
189 |
self._skip_externals = check_yn(None, 'skip-external-entities', |
54847e2c4a50
[pyro source] new option to control wether external entities should be skipped or not
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6582
diff
changeset
|
190 |
source_config.get('skip-external-entities', False)) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
191 |
|
2763
39b42e158249
[ms] proper reset cache on external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2665
diff
changeset
|
192 |
def reset_caches(self): |
39b42e158249
[ms] proper reset cache on external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2665
diff
changeset
|
193 |
"""method called during test to reset potential source caches""" |
5642
6a90357b9769
TimedCache now only accepts values expressed in seconds
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
5426
diff
changeset
|
194 |
self._query_cache = TimedCache(1800) |
2763
39b42e158249
[ms] proper reset cache on external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2665
diff
changeset
|
195 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
196 |
def last_update_time(self): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
197 |
pkey = u'sources.%s.latest-update-time' % self.uri |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
198 |
session = self.repo.internal_session() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
199 |
try: |
6632 | 200 |
rset = session.execute('Any V WHERE X is CWProperty, X value V, X pkey %(k)s', |
201 |
{'k': pkey}) |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
202 |
if not rset: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
203 |
# insert it |
1398
5fe84a5f7035
rename internal entity types to have CW prefix instead of E
sylvain.thenault@logilab.fr
parents:
1263
diff
changeset
|
204 |
session.execute('INSERT CWProperty X: X pkey %(k)s, X value %(v)s', |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
205 |
{'k': pkey, 'v': u'0'}) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
206 |
session.commit() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
207 |
timestamp = 0 |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
208 |
else: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
209 |
assert len(rset) == 1 |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
210 |
timestamp = int(rset[0][0]) |
1016
26387b836099
use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
211 |
return datetime.fromtimestamp(timestamp) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
212 |
finally: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
213 |
session.close() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
214 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
215 |
def init(self): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
216 |
"""method called by the repository once ready to handle request""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
217 |
interval = int(self.config.get('synchronization-interval', 5*60)) |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
218 |
self.repo.looping_task(interval, self.synchronize) |
6309 | 219 |
self.repo.looping_task(self._query_cache.ttl.seconds/10, |
220 |
self._query_cache.clear_expired) |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
221 |
|
6655
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
222 |
def local_eid(self, cnx, extid, session): |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
223 |
etype, dexturi, dextid = cnx.describe(extid) |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
224 |
if dexturi == 'system' or not ( |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
225 |
dexturi in self.repo.sources_by_uri or self._skip_externals): |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
226 |
return self.repo.extid2eid(self, str(extid), etype, session), True |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
227 |
if dexturi in self.repo.sources_by_uri: |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
228 |
source = self.repo.sources_by_uri[dexturi] |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
229 |
cnx = session.pool.connection(source.uri) |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
230 |
eid = source.local_eid(cnx, dextid, session)[0] |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
231 |
return eid, False |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
232 |
return None, None |
6625
54847e2c4a50
[pyro source] new option to control wether external entities should be skipped or not
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6582
diff
changeset
|
233 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
234 |
def synchronize(self, mtime=None): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
235 |
"""synchronize content known by this repository with content in the |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
236 |
external repository |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
237 |
""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
238 |
self.info('synchronizing pyro source %s', self.uri) |
519
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
239 |
cnx = self.get_connection() |
2271
5483155a1e17
avoid additional tb in logs when we can't connect to the external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2269
diff
changeset
|
240 |
try: |
5483155a1e17
avoid additional tb in logs when we can't connect to the external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2269
diff
changeset
|
241 |
extrepo = cnx._repo |
5483155a1e17
avoid additional tb in logs when we can't connect to the external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2269
diff
changeset
|
242 |
except AttributeError: |
5483155a1e17
avoid additional tb in logs when we can't connect to the external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2269
diff
changeset
|
243 |
# fake connection wrapper returned when we can't connect to the |
5483155a1e17
avoid additional tb in logs when we can't connect to the external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2269
diff
changeset
|
244 |
# external source (hence we've no chance to synchronize...) |
5483155a1e17
avoid additional tb in logs when we can't connect to the external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2269
diff
changeset
|
245 |
return |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
246 |
etypes = self.support_entities.keys() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
247 |
if mtime is None: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
248 |
mtime = self.last_update_time() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
249 |
updatetime, modified, deleted = extrepo.entities_modified_since(etypes, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
250 |
mtime) |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
251 |
self._query_cache.clear() |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
252 |
repo = self.repo |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
253 |
session = repo.internal_session() |
4816
c02583cb80a9
repair stuff broken by fti handling changes
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4212
diff
changeset
|
254 |
source = repo.system_source |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
255 |
try: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
256 |
for etype, extid in modified: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
257 |
try: |
6655
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
258 |
eid = self.local_eid(cnx, extid, session) |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
259 |
if eid is not None: |
519
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
260 |
rset = session.eid_rset(eid, etype) |
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
261 |
entity = rset.get_entity(0, 0) |
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
262 |
entity.complete(entity.e_schema.indexable_attributes()) |
4816
c02583cb80a9
repair stuff broken by fti handling changes
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4212
diff
changeset
|
263 |
source.index_entity(session, entity) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
264 |
except: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
265 |
self.exception('while updating %s with external id %s of source %s', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
266 |
etype, extid, self.uri) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
267 |
continue |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
268 |
for etype, extid in deleted: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
269 |
try: |
1952
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1805
diff
changeset
|
270 |
eid = self.extid2eid(str(extid), etype, session, |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1805
diff
changeset
|
271 |
insert=False) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
272 |
# entity has been deleted from external repository but is not known here |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
273 |
if eid is not None: |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4816
diff
changeset
|
274 |
entity = session.entity_from_eid(eid, etype) |
6626
5c20a005bddc
[pyro source] when cleaning relations of an external entity being cleaned up, only delete local relationsd
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6625
diff
changeset
|
275 |
repo.delete_info(session, entity, self.uri, extid, |
5c20a005bddc
[pyro source] when cleaning relations of an external entity being cleaned up, only delete local relationsd
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6625
diff
changeset
|
276 |
scleanup=True) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
277 |
except: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
278 |
self.exception('while updating %s with external id %s of source %s', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
279 |
etype, extid, self.uri) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
280 |
continue |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
281 |
session.execute('SET X value %(v)s WHERE X pkey %(k)s', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
282 |
{'k': u'sources.%s.latest-update-time' % self.uri, |
1016
26387b836099
use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
283 |
'v': unicode(int(mktime(updatetime.timetuple())))}) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
284 |
session.commit() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
285 |
finally: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
286 |
session.close() |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
287 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
288 |
def _get_connection(self): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
289 |
"""open and return a connection to the source""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
290 |
nshost = self.config.get('pyro-ns-host') or self.repo.config['pyro-ns-host'] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
291 |
nsgroup = self.config.get('pyro-ns-group') or self.repo.config['pyro-ns-group'] |
3527
efeb16ff93f3
log when connecting to an external pyro source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3043
diff
changeset
|
292 |
self.info('connecting to instance :%s.%s for user %s', |
3605
1466323ddb9c
fix debug msg
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3527
diff
changeset
|
293 |
nsgroup, self.config['pyro-ns-id'], self.config['cubicweb-user']) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
294 |
#cnxprops = ConnectionProperties(cnxtype=self.config['cnx-type']) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
295 |
return dbapi.connect(database=self.config['pyro-ns-id'], |
2269
0d1b7206fdc9
renamed arg
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1977
diff
changeset
|
296 |
login=self.config['cubicweb-user'], |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
297 |
password=self.config['cubicweb-password'], |
2665
0c6281487f90
[pyro] use lgc.pyro_ext, simplify pyro related options
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2625
diff
changeset
|
298 |
host=nshost, group=nsgroup, |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
299 |
setvreg=False) #cnxprops=cnxprops) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
300 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
301 |
def get_connection(self): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
302 |
try: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
303 |
return self._get_connection() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
304 |
except (ConnectionError, PyroError): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
305 |
self.critical("can't get connection to source %s", self.uri, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
306 |
exc_info=1) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
307 |
return ConnectionWrapper() |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
308 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
309 |
def check_connection(self, cnx): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
310 |
"""check connection validity, return None if the connection is still valid |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
311 |
else a new connection |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
312 |
""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
313 |
# we have to transfer manually thread ownership. This can be done safely |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
314 |
# since the pool to which belong the connection is affected to one |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
315 |
# session/thread and can't be called simultaneously |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
316 |
try: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
317 |
cnx._repo._transferThread(threading.currentThread()) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
318 |
except AttributeError: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
319 |
# inmemory connection |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
320 |
pass |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
321 |
if not isinstance(cnx, ConnectionWrapper): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
322 |
try: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
323 |
cnx.check() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
324 |
return # ok |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
325 |
except (BadConnectionId, ConnectionClosedError): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
326 |
pass |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
327 |
# try to reconnect |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
328 |
return self.get_connection() |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
329 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
330 |
def syntax_tree_search(self, session, union, args=None, cachekey=None, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
331 |
varmap=None): |
2625
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2593
diff
changeset
|
332 |
assert dbg_st_search(self.uri, union, varmap, args, cachekey) |
1238
fa29b5b60107
set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
333 |
rqlkey = union.as_string(kwargs=args) |
fa29b5b60107
set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
334 |
try: |
fa29b5b60107
set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
335 |
results = self._query_cache[rqlkey] |
fa29b5b60107
set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
336 |
except KeyError: |
fa29b5b60107
set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
337 |
results = self._syntax_tree_search(session, union, args) |
fa29b5b60107
set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
338 |
self._query_cache[rqlkey] = results |
2625
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2593
diff
changeset
|
339 |
assert dbg_results(results) |
1238
fa29b5b60107
set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
340 |
return results |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
341 |
|
1238
fa29b5b60107
set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
342 |
def _syntax_tree_search(self, session, union, args): |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
343 |
"""return result from this source for a rql query (actually from a rql |
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
344 |
syntax tree and a solution dictionary mapping each used variable to a |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
345 |
possible type). If cachekey is given, the query necessary to fetch the |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
346 |
results (but not the results themselves) may be cached using this key. |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
347 |
""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
348 |
if not args is None: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
349 |
args = args.copy() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
350 |
# get cached cursor anyway |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
351 |
cu = session.pool[self.uri] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
352 |
if cu is None: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
353 |
# this is a ConnectionWrapper instance |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
354 |
msg = session._("can't connect to source %s, some data may be missing") |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
355 |
session.set_shared_data('sources_error', msg % self.uri) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
356 |
return [] |
6655
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
357 |
translator = RQL2RQL(self) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
358 |
try: |
6655
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
359 |
rql = translator.generate(session, union, args) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
360 |
except UnknownEid, ex: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
361 |
if server.DEBUG: |
2625
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2593
diff
changeset
|
362 |
print ' unknown eid', ex, 'no results' |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
363 |
return [] |
2593
16d9419a4a79
F: start to handle binary debug log level on the server side
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2271
diff
changeset
|
364 |
if server.DEBUG & server.DBG_RQL: |
2625
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2593
diff
changeset
|
365 |
print ' translated rql', rql |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
366 |
try: |
5174
78438ad513ca
#759035: Automate addition of eid cachekey in RQL analysis
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
367 |
rset = cu.execute(rql, args) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
368 |
except Exception, ex: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
369 |
self.exception(str(ex)) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
370 |
msg = session._("error while querying source %s, some data may be missing") |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
371 |
session.set_shared_data('sources_error', msg % self.uri) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
372 |
return [] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
373 |
descr = rset.description |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
374 |
if rset: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
375 |
needtranslation = [] |
519
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
376 |
rows = rset.rows |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
377 |
for i, etype in enumerate(descr[0]): |
3689
deb13e88e037
follow yams 0.25 api changes to improve performance
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3605
diff
changeset
|
378 |
if (etype is None or not self.schema.eschema(etype).final |
3016
5787d1cc8106
[rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2763
diff
changeset
|
379 |
or uidtype(union, i, etype, args)): |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
380 |
needtranslation.append(i) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
381 |
if needtranslation: |
519
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
382 |
cnx = session.pool.connection(self.uri) |
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
383 |
for rowindex in xrange(rset.rowcount - 1, -1, -1): |
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
384 |
row = rows[rowindex] |
6655
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
385 |
localrow = False |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
386 |
for colindex in needtranslation: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
387 |
if row[colindex] is not None: # optional variable |
6655
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
388 |
eid, local = self.local_eid(cnx, row[colindex], session) |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
389 |
if local: |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
390 |
localrow = True |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
391 |
if eid is not None: |
519
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
392 |
row[colindex] = eid |
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
393 |
else: |
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
394 |
# skip this row |
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
395 |
del rows[rowindex] |
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
396 |
del descr[rowindex] |
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
397 |
break |
6655
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
398 |
else: |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
399 |
# skip row if it only contains eids of entities which |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
400 |
# are actually from a source we also know locally, |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
401 |
# except if some args specified (XXX should actually |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
402 |
# check if there are some args local to the source) |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
403 |
if not (translator.has_local_eid or localrow): |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
404 |
del rows[rowindex] |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
405 |
del descr[rowindex] |
519
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
406 |
results = rows |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
407 |
else: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
408 |
results = [] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
409 |
return results |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
410 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
411 |
def _entity_relations_and_kwargs(self, session, entity): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
412 |
relations = [] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
413 |
kwargs = {'x': self.eid2extid(entity.eid, session)} |
6652
592c88c8f018
[3.10] use cw_attr_cache, entity has no more iteritems method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6632
diff
changeset
|
414 |
for key, val in entity.cw_attr_cache.iteritems(): |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
415 |
relations.append('X %s %%(%s)s' % (key, key)) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
416 |
kwargs[key] = val |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
417 |
return relations, kwargs |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
418 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
419 |
def add_entity(self, session, entity): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
420 |
"""add a new entity to the source""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
421 |
raise NotImplementedError() |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
422 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
423 |
def update_entity(self, session, entity): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
424 |
"""update an entity in the source""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
425 |
relations, kwargs = self._entity_relations_and_kwargs(session, entity) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
426 |
cu = session.pool[self.uri] |
5363
d336476dcf57
[3.8 api update] remaining cache keys
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5323
diff
changeset
|
427 |
cu.execute('SET %s WHERE X eid %%(x)s' % ','.join(relations), kwargs) |
1805 | 428 |
self._query_cache.clear() |
3043
7bcb7f1c0f46
force cache clear when external entity is edited, we don't know what may occurs in external repository hooks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3016
diff
changeset
|
429 |
entity.clear_all_caches() |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
430 |
|
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4816
diff
changeset
|
431 |
def delete_entity(self, session, entity): |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
432 |
"""delete an entity from the source""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
433 |
cu = session.pool[self.uri] |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4816
diff
changeset
|
434 |
cu.execute('DELETE %s X WHERE X eid %%(x)s' % entity.__regid__, |
5363
d336476dcf57
[3.8 api update] remaining cache keys
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5323
diff
changeset
|
435 |
{'x': self.eid2extid(entity.eid, session)}) |
1805 | 436 |
self._query_cache.clear() |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
437 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
438 |
def add_relation(self, session, subject, rtype, object): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
439 |
"""add a relation to the source""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
440 |
cu = session.pool[self.uri] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
441 |
cu.execute('SET X %s Y WHERE X eid %%(x)s, Y eid %%(y)s' % rtype, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
442 |
{'x': self.eid2extid(subject, session), |
5363
d336476dcf57
[3.8 api update] remaining cache keys
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5323
diff
changeset
|
443 |
'y': self.eid2extid(object, session)}) |
1805 | 444 |
self._query_cache.clear() |
3043
7bcb7f1c0f46
force cache clear when external entity is edited, we don't know what may occurs in external repository hooks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3016
diff
changeset
|
445 |
session.entity_from_eid(subject).clear_all_caches() |
7bcb7f1c0f46
force cache clear when external entity is edited, we don't know what may occurs in external repository hooks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3016
diff
changeset
|
446 |
session.entity_from_eid(object).clear_all_caches() |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
447 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
448 |
def delete_relation(self, session, subject, rtype, object): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
449 |
"""delete a relation from the source""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
450 |
cu = session.pool[self.uri] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
451 |
cu.execute('DELETE X %s Y WHERE X eid %%(x)s, Y eid %%(y)s' % rtype, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
452 |
{'x': self.eid2extid(subject, session), |
5363
d336476dcf57
[3.8 api update] remaining cache keys
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5323
diff
changeset
|
453 |
'y': self.eid2extid(object, session)}) |
1805 | 454 |
self._query_cache.clear() |
3043
7bcb7f1c0f46
force cache clear when external entity is edited, we don't know what may occurs in external repository hooks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3016
diff
changeset
|
455 |
session.entity_from_eid(subject).clear_all_caches() |
7bcb7f1c0f46
force cache clear when external entity is edited, we don't know what may occurs in external repository hooks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3016
diff
changeset
|
456 |
session.entity_from_eid(object).clear_all_caches() |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
457 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
458 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
459 |
class RQL2RQL(object): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
460 |
"""translate a local rql query to be executed on a distant repository""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
461 |
def __init__(self, source): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
462 |
self.source = source |
390
739d12586b9d
avoid potentialy mis-interpreted attribute error
sylvain.thenault@logilab.fr
parents:
257
diff
changeset
|
463 |
self.current_operator = None |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
464 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
465 |
def _accept_children(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
466 |
res = [] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
467 |
for child in node.children: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
468 |
rql = child.accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
469 |
if rql is not None: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
470 |
res.append(rql) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
471 |
return res |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
472 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
473 |
def generate(self, session, rqlst, args): |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
474 |
self._session = session |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
475 |
self.kwargs = args |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
476 |
self.need_translation = False |
6655
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
477 |
self.has_local_eid = False |
5174
78438ad513ca
#759035: Automate addition of eid cachekey in RQL analysis
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
478 |
return self.visit_union(rqlst) |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
479 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
480 |
def visit_union(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
481 |
s = self._accept_children(node) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
482 |
if len(s) > 1: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
483 |
return ' UNION '.join('(%s)' % q for q in s) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
484 |
return s[0] |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
485 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
486 |
def visit_select(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
487 |
"""return the tree as an encoded rql string""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
488 |
self._varmaker = rqlvar_maker(defined=node.defined_vars.copy()) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
489 |
self._const_var = {} |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
490 |
if node.distinct: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
491 |
base = 'DISTINCT Any' |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
492 |
else: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
493 |
base = 'Any' |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
494 |
s = ['%s %s' % (base, ','.join(v.accept(self) for v in node.selection))] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
495 |
if node.groupby: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
496 |
s.append('GROUPBY %s' % ', '.join(group.accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
497 |
for group in node.groupby)) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
498 |
if node.orderby: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
499 |
s.append('ORDERBY %s' % ', '.join(self.visit_sortterm(term) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
500 |
for term in node.orderby)) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
501 |
if node.limit is not None: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
502 |
s.append('LIMIT %s' % node.limit) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
503 |
if node.offset: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
504 |
s.append('OFFSET %s' % node.offset) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
505 |
restrictions = [] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
506 |
if node.where is not None: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
507 |
nr = node.where.accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
508 |
if nr is not None: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
509 |
restrictions.append(nr) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
510 |
if restrictions: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
511 |
s.append('WHERE %s' % ','.join(restrictions)) |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
512 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
513 |
if node.having: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
514 |
s.append('HAVING %s' % ', '.join(term.accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
515 |
for term in node.having)) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
516 |
subqueries = [] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
517 |
for subquery in node.with_: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
518 |
subqueries.append('%s BEING (%s)' % (','.join(ca.name for ca in subquery.aliases), |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
519 |
self.visit_union(subquery.query))) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
520 |
if subqueries: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
521 |
s.append('WITH %s' % (','.join(subqueries))) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
522 |
return ' '.join(s) |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
523 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
524 |
def visit_and(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
525 |
res = self._accept_children(node) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
526 |
if res: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
527 |
return ', '.join(res) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
528 |
return |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
529 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
530 |
def visit_or(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
531 |
res = self._accept_children(node) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
532 |
if len(res) > 1: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
533 |
return ' OR '.join('(%s)' % rql for rql in res) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
534 |
elif res: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
535 |
return res[0] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
536 |
return |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
537 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
538 |
def visit_not(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
539 |
rql = node.children[0].accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
540 |
if rql: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
541 |
return 'NOT (%s)' % rql |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
542 |
return |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
543 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
544 |
def visit_exists(self, node): |
5582
3e133b29a1a4
[rql2sql] follow rql 0.26.1 changes: NOT nodes normalization, allowing simplification of sql generation, and fix #XXX
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5426
diff
changeset
|
545 |
rql = node.children[0].accept(self) |
3e133b29a1a4
[rql2sql] follow rql 0.26.1 changes: NOT nodes normalization, allowing simplification of sql generation, and fix #XXX
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5426
diff
changeset
|
546 |
if rql: |
3e133b29a1a4
[rql2sql] follow rql 0.26.1 changes: NOT nodes normalization, allowing simplification of sql generation, and fix #XXX
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5426
diff
changeset
|
547 |
return 'EXISTS(%s)' % rql |
3e133b29a1a4
[rql2sql] follow rql 0.26.1 changes: NOT nodes normalization, allowing simplification of sql generation, and fix #XXX
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5426
diff
changeset
|
548 |
return |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
549 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
550 |
def visit_relation(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
551 |
try: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
552 |
if isinstance(node.children[0], Constant): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
553 |
# simplified rqlst, reintroduce eid relation |
391
42693fe3ef6f
skip neged relation here as well
sylvain.thenault@logilab.fr
parents:
390
diff
changeset
|
554 |
try: |
42693fe3ef6f
skip neged relation here as well
sylvain.thenault@logilab.fr
parents:
390
diff
changeset
|
555 |
restr, lhs = self.process_eid_const(node.children[0]) |
42693fe3ef6f
skip neged relation here as well
sylvain.thenault@logilab.fr
parents:
390
diff
changeset
|
556 |
except UnknownEid: |
42693fe3ef6f
skip neged relation here as well
sylvain.thenault@logilab.fr
parents:
390
diff
changeset
|
557 |
# can safely skip not relation with an unsupported eid |
5582
3e133b29a1a4
[rql2sql] follow rql 0.26.1 changes: NOT nodes normalization, allowing simplification of sql generation, and fix #XXX
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5426
diff
changeset
|
558 |
if neged_relation(node): |
391
42693fe3ef6f
skip neged relation here as well
sylvain.thenault@logilab.fr
parents:
390
diff
changeset
|
559 |
return |
42693fe3ef6f
skip neged relation here as well
sylvain.thenault@logilab.fr
parents:
390
diff
changeset
|
560 |
raise |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
561 |
else: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
562 |
lhs = node.children[0].accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
563 |
restr = None |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
564 |
except UnknownEid: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
565 |
# can safely skip not relation with an unsupported eid |
5582
3e133b29a1a4
[rql2sql] follow rql 0.26.1 changes: NOT nodes normalization, allowing simplification of sql generation, and fix #XXX
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5426
diff
changeset
|
566 |
if neged_relation(node): |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
567 |
return |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
568 |
# XXX what about optional relation or outer NOT EXISTS() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
569 |
raise |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
570 |
if node.optional in ('left', 'both'): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
571 |
lhs += '?' |
3689
deb13e88e037
follow yams 0.25 api changes to improve performance
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3605
diff
changeset
|
572 |
if node.r_type == 'eid' or not self.source.schema.rschema(node.r_type).final: |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
573 |
self.need_translation = True |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
574 |
self.current_operator = node.operator() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
575 |
if isinstance(node.children[0], Constant): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
576 |
self.current_etypes = (node.children[0].uidtype,) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
577 |
else: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
578 |
self.current_etypes = node.children[0].variable.stinfo['possibletypes'] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
579 |
try: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
580 |
rhs = node.children[1].accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
581 |
except UnknownEid: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
582 |
# can safely skip not relation with an unsupported eid |
5582
3e133b29a1a4
[rql2sql] follow rql 0.26.1 changes: NOT nodes normalization, allowing simplification of sql generation, and fix #XXX
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5426
diff
changeset
|
583 |
if neged_relation(node): |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
584 |
return |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
585 |
# XXX what about optional relation or outer NOT EXISTS() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
586 |
raise |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
587 |
except ReplaceByInOperator, ex: |
1952
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1805
diff
changeset
|
588 |
rhs = 'IN (%s)' % ','.join(eid for eid in ex.eids) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
589 |
self.need_translation = False |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
590 |
self.current_operator = None |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
591 |
if node.optional in ('right', 'both'): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
592 |
rhs += '?' |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
593 |
if restr is not None: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
594 |
return '%s %s %s, %s' % (lhs, node.r_type, rhs, restr) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
595 |
return '%s %s %s' % (lhs, node.r_type, rhs) |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
596 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
597 |
def visit_comparison(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
598 |
if node.operator in ('=', 'IS'): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
599 |
return node.children[0].accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
600 |
return '%s %s' % (node.operator.encode(), |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
601 |
node.children[0].accept(self)) |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
602 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
603 |
def visit_mathexpression(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
604 |
return '(%s %s %s)' % (node.children[0].accept(self), |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
605 |
node.operator.encode(), |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
606 |
node.children[1].accept(self)) |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
607 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
608 |
def visit_function(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
609 |
#if node.name == 'IN': |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
610 |
res = [] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
611 |
for child in node.children: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
612 |
try: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
613 |
rql = child.accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
614 |
except UnknownEid, ex: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
615 |
continue |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
616 |
res.append(rql) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
617 |
if not res: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
618 |
raise ex |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
619 |
return '%s(%s)' % (node.name, ', '.join(res)) |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
620 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
621 |
def visit_constant(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
622 |
if self.need_translation or node.uidtype: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
623 |
if node.type == 'Int': |
6655
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
624 |
self.has_local_eid = True |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
625 |
return str(self.eid2extid(node.value)) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
626 |
if node.type == 'Substitute': |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
627 |
key = node.value |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
628 |
# ensure we have not yet translated the value... |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
629 |
if not key in self._const_var: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
630 |
self.kwargs[key] = self.eid2extid(self.kwargs[key]) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
631 |
self._const_var[key] = None |
6655
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
632 |
self.has_local_eid = True |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
633 |
return node.as_string() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
634 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
635 |
def visit_variableref(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
636 |
"""get the sql name for a variable reference""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
637 |
return node.name |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
638 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
639 |
def visit_sortterm(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
640 |
if node.asc: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
641 |
return node.term.accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
642 |
return '%s DESC' % node.term.accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
643 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
644 |
def process_eid_const(self, const): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
645 |
value = const.eval(self.kwargs) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
646 |
try: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
647 |
return None, self._const_var[value] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
648 |
except: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
649 |
var = self._varmaker.next() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
650 |
self.need_translation = True |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
651 |
restr = '%s eid %s' % (var, self.visit_constant(const)) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
652 |
self.need_translation = False |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
653 |
self._const_var[value] = var |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
654 |
return restr, var |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
655 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
656 |
def eid2extid(self, eid): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
657 |
try: |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
658 |
return self.source.eid2extid(eid, self._session) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
659 |
except UnknownEid: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
660 |
operator = self.current_operator |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
661 |
if operator is not None and operator != '=': |
1954 | 662 |
# deal with query like "X eid > 12" |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
663 |
# |
1954 | 664 |
# The problem is that eid order in the external source may |
665 |
# differ from the local source |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
666 |
# |
1954 | 667 |
# So search for all eids from this source matching the condition |
668 |
# locally and then to replace the "> 12" branch by "IN (eids)" |
|
669 |
# |
|
670 |
# XXX we may have to insert a huge number of eids...) |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
671 |
sql = "SELECT extid FROM entities WHERE source='%s' AND type IN (%s) AND eid%s%s" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
672 |
etypes = ','.join("'%s'" % etype for etype in self.current_etypes) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
673 |
cu = self._session.system_sql(sql % (self.source.uri, etypes, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
674 |
operator, eid)) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
675 |
# XXX buggy cu.rowcount which may be zero while there are some |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
676 |
# results |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
677 |
rows = cu.fetchall() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
678 |
if rows: |
1952
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1805
diff
changeset
|
679 |
raise ReplaceByInOperator((b64decode(r[0]) for r in rows)) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
680 |
raise |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
681 |