author | Sylvain Thénault <sylvain.thenault@logilab.fr> |
Wed, 15 Sep 2010 18:43:14 +0200 | |
branch | stable |
changeset 6251 | afc757568492 |
parent 6130 | 15fa8425b6e7 |
child 6309 | 9f03e3c32676 |
permissions | -rw-r--r-- |
5421
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
1 |
# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
2 |
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
3 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
4 |
# This file is part of CubicWeb. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
5 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
6 |
# CubicWeb is free software: you can redistribute it and/or modify it under the |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
7 |
# terms of the GNU Lesser General Public License as published by the Free |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
8 |
# Software Foundation, either version 2.1 of the License, or (at your option) |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
9 |
# any later version. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
10 |
# |
5424
8ecbcbff9777
replace logilab-common by CubicWeb in disclaimer
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5421
diff
changeset
|
11 |
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT |
5421
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
12 |
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
13 |
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
14 |
# details. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
15 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
16 |
# You should have received a copy of the GNU Lesser General Public License along |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
17 |
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>. |
6128 | 18 |
"""Source to query another RQL repository using pyro""" |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
19 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
20 |
__docformat__ = "restructuredtext en" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
21 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
22 |
import threading |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
23 |
from os.path import join |
1016
26387b836099
use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
24 |
from time import mktime |
26387b836099
use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
25 |
from datetime import datetime |
1952
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1805
diff
changeset
|
26 |
from base64 import b64decode |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
27 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
28 |
from Pyro.errors import PyroError, ConnectionClosedError |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
29 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
30 |
from logilab.common.configuration import REQUIRED |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
31 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
32 |
from rql.nodes import Constant |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
33 |
from rql.utils import rqlvar_maker |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
34 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
35 |
from cubicweb import dbapi, server |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
36 |
from cubicweb import BadConnectionId, UnknownEid, ConnectionError |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
37 |
from cubicweb.cwconfig import register_persistent_options |
2625
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2593
diff
changeset
|
38 |
from cubicweb.server.sources import (AbstractSource, ConnectionWrapper, |
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2593
diff
changeset
|
39 |
TimedCache, dbg_st_search, dbg_results) |
5582
3e133b29a1a4
[rql2sql] follow rql 0.26.1 changes: NOT nodes normalization, allowing simplification of sql generation, and fix #XXX
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5426
diff
changeset
|
40 |
from cubicweb.server.msplanner import neged_relation |
3016
5787d1cc8106
[rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2763
diff
changeset
|
41 |
|
5787d1cc8106
[rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2763
diff
changeset
|
42 |
def uidtype(union, col, etype, args): |
5787d1cc8106
[rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2763
diff
changeset
|
43 |
select, col = union.locate_subquery(col, etype, args) |
5787d1cc8106
[rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2763
diff
changeset
|
44 |
return getattr(select.selection[col], 'uidtype', None) |
5787d1cc8106
[rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2763
diff
changeset
|
45 |
|
6130
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
46 |
def load_mapping_file(mappingfile): |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
47 |
mapping = {} |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
48 |
execfile(mappingfile, mapping) |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
49 |
for junk in ('__builtins__', '__doc__'): |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
50 |
mapping.pop(junk, None) |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
51 |
mapping.setdefault('support_relations', {}) |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
52 |
mapping.setdefault('dont_cross_relations', set()) |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
53 |
mapping.setdefault('cross_relations', set()) |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
54 |
|
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
55 |
# do some basic checks of the mapping content |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
56 |
assert 'support_entities' in mapping, \ |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
57 |
'mapping file should at least define support_entities' |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
58 |
assert isinstance(mapping['support_entities'], dict) |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
59 |
assert isinstance(mapping['support_relations'], dict) |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
60 |
assert isinstance(mapping['dont_cross_relations'], set) |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
61 |
assert isinstance(mapping['cross_relations'], set) |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
62 |
unknown = set(mapping) - set( ('support_entities', 'support_relations', |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
63 |
'dont_cross_relations', 'cross_relations') ) |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
64 |
assert not unknown, 'unknown mapping attribute(s): %s' % unknown |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
65 |
# relations that are necessarily not crossed |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
66 |
mapping['dont_cross_relations'] |= set(('owned_by', 'created_by')) |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
67 |
for rtype in ('is', 'is_instance_of'): |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
68 |
assert rtype not in mapping['dont_cross_relations'], \ |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
69 |
'%s relation should not be in dont_cross_relations' % rtype |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
70 |
assert rtype not in mapping['support_relations'], \ |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
71 |
'%s relation should not be in support_relations' % rtype |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
72 |
return mapping |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
73 |
|
3016
5787d1cc8106
[rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2763
diff
changeset
|
74 |
|
1138
22f634977c95
make pylint happy, fix some bugs on the way
sylvain.thenault@logilab.fr
parents:
1016
diff
changeset
|
75 |
class ReplaceByInOperator(Exception): |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
76 |
def __init__(self, eids): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
77 |
self.eids = eids |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
78 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
79 |
class PyroRQLSource(AbstractSource): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
80 |
"""External repository source, using Pyro connection""" |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
81 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
82 |
# boolean telling if modification hooks should be called when something is |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
83 |
# modified in this source |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
84 |
should_call_hooks = False |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
85 |
# boolean telling if the repository should connect to this source during |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
86 |
# migration |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
87 |
connect_for_migration = False |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
88 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
89 |
options = ( |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
90 |
# XXX pyro-ns host/port |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
91 |
('pyro-ns-id', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
92 |
{'type' : 'string', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
93 |
'default': REQUIRED, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
94 |
'help': 'identifier of the repository in the pyro name server', |
5323
329b4f6d18b4
[config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5174
diff
changeset
|
95 |
'group': 'pyro-source', 'level': 0, |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
96 |
}), |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
97 |
('mapping-file', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
98 |
{'type' : 'string', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
99 |
'default': REQUIRED, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
100 |
'help': 'path to a python file with the schema mapping definition', |
5323
329b4f6d18b4
[config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5174
diff
changeset
|
101 |
'group': 'pyro-source', 'level': 1, |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
102 |
}), |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
103 |
('cubicweb-user', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
104 |
{'type' : 'string', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
105 |
'default': REQUIRED, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
106 |
'help': 'user to use for connection on the distant repository', |
5323
329b4f6d18b4
[config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5174
diff
changeset
|
107 |
'group': 'pyro-source', 'level': 0, |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
108 |
}), |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
109 |
('cubicweb-password', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
110 |
{'type' : 'password', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
111 |
'default': '', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
112 |
'help': 'user to use for connection on the distant repository', |
5323
329b4f6d18b4
[config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5174
diff
changeset
|
113 |
'group': 'pyro-source', 'level': 0, |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
114 |
}), |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
115 |
('base-url', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
116 |
{'type' : 'string', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
117 |
'default': '', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
118 |
'help': 'url of the web site for the distant repository, if you want ' |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
119 |
'to generate external link to entities from this repository', |
5323
329b4f6d18b4
[config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5174
diff
changeset
|
120 |
'group': 'pyro-source', 'level': 1, |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
121 |
}), |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
122 |
('pyro-ns-host', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
123 |
{'type' : 'string', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
124 |
'default': None, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
125 |
'help': 'Pyro name server\'s host. If not set, default to the value \ |
2665
0c6281487f90
[pyro] use lgc.pyro_ext, simplify pyro related options
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2625
diff
changeset
|
126 |
from all_in_one.conf. It may contains port information using <host>:<port> notation.', |
5323
329b4f6d18b4
[config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5174
diff
changeset
|
127 |
'group': 'pyro-source', 'level': 1, |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
128 |
}), |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
129 |
('pyro-ns-group', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
130 |
{'type' : 'string', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
131 |
'default': None, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
132 |
'help': 'Pyro name server\'s group where the repository will be \ |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
133 |
registered. If not set, default to the value from all_in_one.conf.', |
5323
329b4f6d18b4
[config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5174
diff
changeset
|
134 |
'group': 'pyro-source', 'level': 1, |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
135 |
}), |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
136 |
('synchronization-interval', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
137 |
{'type' : 'int', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
138 |
'default': 5*60, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
139 |
'help': 'interval between synchronization with the external \ |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
140 |
repository (default to 5 minutes).', |
5323
329b4f6d18b4
[config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5174
diff
changeset
|
141 |
'group': 'pyro-source', 'level': 2, |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
142 |
}), |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
143 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
144 |
) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
145 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
146 |
PUBLIC_KEYS = AbstractSource.PUBLIC_KEYS + ('base-url',) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
147 |
_conn = None |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
148 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
149 |
def __init__(self, repo, appschema, source_config, *args, **kwargs): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
150 |
AbstractSource.__init__(self, repo, appschema, source_config, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
151 |
*args, **kwargs) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
152 |
mappingfile = source_config['mapping-file'] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
153 |
if not mappingfile[0] == '/': |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
154 |
mappingfile = join(repo.config.apphome, mappingfile) |
6130
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
155 |
mapping = load_mapping_file(mappingfile) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
156 |
self.support_entities = mapping['support_entities'] |
6130
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
157 |
self.support_relations = mapping['support_relations'] |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
158 |
self.dont_cross_relations = mapping['dont_cross_relations'] |
15fa8425b6e7
oops, missing part of 6125:628cf5213154 (mapping file checking)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
159 |
self.cross_relations = mapping['cross_relations'] |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
160 |
baseurl = source_config.get('base-url') |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
161 |
if baseurl and not baseurl.endswith('/'): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
162 |
source_config['base-url'] += '/' |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
163 |
self.config = source_config |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
164 |
myoptions = (('%s.latest-update-time' % self.uri, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
165 |
{'type' : 'int', 'sitewide': True, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
166 |
'default': 0, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
167 |
'help': _('timestamp of the latest source synchronization.'), |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
168 |
'group': 'sources', |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
169 |
}),) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
170 |
register_persistent_options(myoptions) |
5642
6a90357b9769
TimedCache now only accepts values expressed in seconds
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
5426
diff
changeset
|
171 |
self._query_cache = TimedCache(1800) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
172 |
|
2763
39b42e158249
[ms] proper reset cache on external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2665
diff
changeset
|
173 |
def reset_caches(self): |
39b42e158249
[ms] proper reset cache on external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2665
diff
changeset
|
174 |
"""method called during test to reset potential source caches""" |
5642
6a90357b9769
TimedCache now only accepts values expressed in seconds
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
5426
diff
changeset
|
175 |
self._query_cache = TimedCache(1800) |
2763
39b42e158249
[ms] proper reset cache on external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2665
diff
changeset
|
176 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
177 |
def last_update_time(self): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
178 |
pkey = u'sources.%s.latest-update-time' % self.uri |
1398
5fe84a5f7035
rename internal entity types to have CW prefix instead of E
sylvain.thenault@logilab.fr
parents:
1263
diff
changeset
|
179 |
rql = 'Any V WHERE X is CWProperty, X value V, X pkey %(k)s' |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
180 |
session = self.repo.internal_session() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
181 |
try: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
182 |
rset = session.execute(rql, {'k': pkey}) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
183 |
if not rset: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
184 |
# insert it |
1398
5fe84a5f7035
rename internal entity types to have CW prefix instead of E
sylvain.thenault@logilab.fr
parents:
1263
diff
changeset
|
185 |
session.execute('INSERT CWProperty X: X pkey %(k)s, X value %(v)s', |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
186 |
{'k': pkey, 'v': u'0'}) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
187 |
session.commit() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
188 |
timestamp = 0 |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
189 |
else: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
190 |
assert len(rset) == 1 |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
191 |
timestamp = int(rset[0][0]) |
1016
26387b836099
use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
192 |
return datetime.fromtimestamp(timestamp) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
193 |
finally: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
194 |
session.close() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
195 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
196 |
def init(self): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
197 |
"""method called by the repository once ready to handle request""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
198 |
interval = int(self.config.get('synchronization-interval', 5*60)) |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
199 |
self.repo.looping_task(interval, self.synchronize) |
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
200 |
self.repo.looping_task(self._query_cache.ttl.seconds/10, self._query_cache.clear_expired) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
201 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
202 |
def synchronize(self, mtime=None): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
203 |
"""synchronize content known by this repository with content in the |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
204 |
external repository |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
205 |
""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
206 |
self.info('synchronizing pyro source %s', self.uri) |
519
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
207 |
cnx = self.get_connection() |
2271
5483155a1e17
avoid additional tb in logs when we can't connect to the external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2269
diff
changeset
|
208 |
try: |
5483155a1e17
avoid additional tb in logs when we can't connect to the external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2269
diff
changeset
|
209 |
extrepo = cnx._repo |
5483155a1e17
avoid additional tb in logs when we can't connect to the external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2269
diff
changeset
|
210 |
except AttributeError: |
5483155a1e17
avoid additional tb in logs when we can't connect to the external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2269
diff
changeset
|
211 |
# fake connection wrapper returned when we can't connect to the |
5483155a1e17
avoid additional tb in logs when we can't connect to the external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2269
diff
changeset
|
212 |
# external source (hence we've no chance to synchronize...) |
5483155a1e17
avoid additional tb in logs when we can't connect to the external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2269
diff
changeset
|
213 |
return |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
214 |
etypes = self.support_entities.keys() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
215 |
if mtime is None: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
216 |
mtime = self.last_update_time() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
217 |
updatetime, modified, deleted = extrepo.entities_modified_since(etypes, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
218 |
mtime) |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
219 |
self._query_cache.clear() |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
220 |
repo = self.repo |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
221 |
session = repo.internal_session() |
4816
c02583cb80a9
repair stuff broken by fti handling changes
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4212
diff
changeset
|
222 |
source = repo.system_source |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
223 |
try: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
224 |
for etype, extid in modified: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
225 |
try: |
519
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
226 |
exturi = cnx.describe(extid)[1] |
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
227 |
if exturi == 'system' or not exturi in repo.sources_by_uri: |
1952
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1805
diff
changeset
|
228 |
eid = self.extid2eid(str(extid), etype, session) |
519
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
229 |
rset = session.eid_rset(eid, etype) |
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
230 |
entity = rset.get_entity(0, 0) |
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
231 |
entity.complete(entity.e_schema.indexable_attributes()) |
4816
c02583cb80a9
repair stuff broken by fti handling changes
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4212
diff
changeset
|
232 |
source.index_entity(session, entity) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
233 |
except: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
234 |
self.exception('while updating %s with external id %s of source %s', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
235 |
etype, extid, self.uri) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
236 |
continue |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
237 |
for etype, extid in deleted: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
238 |
try: |
1952
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1805
diff
changeset
|
239 |
eid = self.extid2eid(str(extid), etype, session, |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1805
diff
changeset
|
240 |
insert=False) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
241 |
# entity has been deleted from external repository but is not known here |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
242 |
if eid is not None: |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4816
diff
changeset
|
243 |
entity = session.entity_from_eid(eid, etype) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4816
diff
changeset
|
244 |
repo.delete_info(session, entity, self.uri, extid) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
245 |
except: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
246 |
self.exception('while updating %s with external id %s of source %s', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
247 |
etype, extid, self.uri) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
248 |
continue |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
249 |
session.execute('SET X value %(v)s WHERE X pkey %(k)s', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
250 |
{'k': u'sources.%s.latest-update-time' % self.uri, |
1016
26387b836099
use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
251 |
'v': unicode(int(mktime(updatetime.timetuple())))}) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
252 |
session.commit() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
253 |
finally: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
254 |
session.close() |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
255 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
256 |
def _get_connection(self): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
257 |
"""open and return a connection to the source""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
258 |
nshost = self.config.get('pyro-ns-host') or self.repo.config['pyro-ns-host'] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
259 |
nsgroup = self.config.get('pyro-ns-group') or self.repo.config['pyro-ns-group'] |
3527
efeb16ff93f3
log when connecting to an external pyro source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3043
diff
changeset
|
260 |
self.info('connecting to instance :%s.%s for user %s', |
3605
1466323ddb9c
fix debug msg
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3527
diff
changeset
|
261 |
nsgroup, self.config['pyro-ns-id'], self.config['cubicweb-user']) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
262 |
#cnxprops = ConnectionProperties(cnxtype=self.config['cnx-type']) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
263 |
return dbapi.connect(database=self.config['pyro-ns-id'], |
2269
0d1b7206fdc9
renamed arg
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1977
diff
changeset
|
264 |
login=self.config['cubicweb-user'], |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
265 |
password=self.config['cubicweb-password'], |
2665
0c6281487f90
[pyro] use lgc.pyro_ext, simplify pyro related options
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2625
diff
changeset
|
266 |
host=nshost, group=nsgroup, |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
267 |
setvreg=False) #cnxprops=cnxprops) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
268 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
269 |
def get_connection(self): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
270 |
try: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
271 |
return self._get_connection() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
272 |
except (ConnectionError, PyroError): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
273 |
self.critical("can't get connection to source %s", self.uri, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
274 |
exc_info=1) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
275 |
return ConnectionWrapper() |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
276 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
277 |
def check_connection(self, cnx): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
278 |
"""check connection validity, return None if the connection is still valid |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
279 |
else a new connection |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
280 |
""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
281 |
# we have to transfer manually thread ownership. This can be done safely |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
282 |
# since the pool to which belong the connection is affected to one |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
283 |
# session/thread and can't be called simultaneously |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
284 |
try: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
285 |
cnx._repo._transferThread(threading.currentThread()) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
286 |
except AttributeError: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
287 |
# inmemory connection |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
288 |
pass |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
289 |
if not isinstance(cnx, ConnectionWrapper): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
290 |
try: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
291 |
cnx.check() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
292 |
return # ok |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
293 |
except (BadConnectionId, ConnectionClosedError): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
294 |
pass |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
295 |
# try to reconnect |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
296 |
return self.get_connection() |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
297 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
298 |
def syntax_tree_search(self, session, union, args=None, cachekey=None, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
299 |
varmap=None): |
2625
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2593
diff
changeset
|
300 |
assert dbg_st_search(self.uri, union, varmap, args, cachekey) |
1238
fa29b5b60107
set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
301 |
rqlkey = union.as_string(kwargs=args) |
fa29b5b60107
set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
302 |
try: |
fa29b5b60107
set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
303 |
results = self._query_cache[rqlkey] |
fa29b5b60107
set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
304 |
except KeyError: |
fa29b5b60107
set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
305 |
results = self._syntax_tree_search(session, union, args) |
fa29b5b60107
set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
306 |
self._query_cache[rqlkey] = results |
2625
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2593
diff
changeset
|
307 |
assert dbg_results(results) |
1238
fa29b5b60107
set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
308 |
return results |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
309 |
|
1238
fa29b5b60107
set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
310 |
def _syntax_tree_search(self, session, union, args): |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
311 |
"""return result from this source for a rql query (actually from a rql |
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
312 |
syntax tree and a solution dictionary mapping each used variable to a |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
313 |
possible type). If cachekey is given, the query necessary to fetch the |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
314 |
results (but not the results themselves) may be cached using this key. |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
315 |
""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
316 |
if not args is None: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
317 |
args = args.copy() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
318 |
# get cached cursor anyway |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
319 |
cu = session.pool[self.uri] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
320 |
if cu is None: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
321 |
# this is a ConnectionWrapper instance |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
322 |
msg = session._("can't connect to source %s, some data may be missing") |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
323 |
session.set_shared_data('sources_error', msg % self.uri) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
324 |
return [] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
325 |
try: |
5174
78438ad513ca
#759035: Automate addition of eid cachekey in RQL analysis
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
326 |
rql = RQL2RQL(self).generate(session, union, args) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
327 |
except UnknownEid, ex: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
328 |
if server.DEBUG: |
2625
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2593
diff
changeset
|
329 |
print ' unknown eid', ex, 'no results' |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
330 |
return [] |
2593
16d9419a4a79
F: start to handle binary debug log level on the server side
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2271
diff
changeset
|
331 |
if server.DEBUG & server.DBG_RQL: |
2625
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2593
diff
changeset
|
332 |
print ' translated rql', rql |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
333 |
try: |
5174
78438ad513ca
#759035: Automate addition of eid cachekey in RQL analysis
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
334 |
rset = cu.execute(rql, args) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
335 |
except Exception, ex: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
336 |
self.exception(str(ex)) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
337 |
msg = session._("error while querying source %s, some data may be missing") |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
338 |
session.set_shared_data('sources_error', msg % self.uri) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
339 |
return [] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
340 |
descr = rset.description |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
341 |
if rset: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
342 |
needtranslation = [] |
519
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
343 |
rows = rset.rows |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
344 |
for i, etype in enumerate(descr[0]): |
3689
deb13e88e037
follow yams 0.25 api changes to improve performance
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3605
diff
changeset
|
345 |
if (etype is None or not self.schema.eschema(etype).final |
3016
5787d1cc8106
[rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2763
diff
changeset
|
346 |
or uidtype(union, i, etype, args)): |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
347 |
needtranslation.append(i) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
348 |
if needtranslation: |
519
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
349 |
cnx = session.pool.connection(self.uri) |
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
350 |
for rowindex in xrange(rset.rowcount - 1, -1, -1): |
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
351 |
row = rows[rowindex] |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
352 |
for colindex in needtranslation: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
353 |
if row[colindex] is not None: # optional variable |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
354 |
etype = descr[rowindex][colindex] |
519
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
355 |
exttype, exturi, extid = cnx.describe(row[colindex]) |
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
356 |
if exturi == 'system' or not exturi in self.repo.sources_by_uri: |
1952
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1805
diff
changeset
|
357 |
eid = self.extid2eid(str(row[colindex]), etype, |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1805
diff
changeset
|
358 |
session) |
519
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
359 |
row[colindex] = eid |
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
360 |
else: |
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
361 |
# skip this row |
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
362 |
del rows[rowindex] |
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
363 |
del descr[rowindex] |
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
364 |
break |
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
365 |
results = rows |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
366 |
else: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
367 |
results = [] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
368 |
return results |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
369 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
370 |
def _entity_relations_and_kwargs(self, session, entity): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
371 |
relations = [] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
372 |
kwargs = {'x': self.eid2extid(entity.eid, session)} |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
373 |
for key, val in entity.iteritems(): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
374 |
relations.append('X %s %%(%s)s' % (key, key)) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
375 |
kwargs[key] = val |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
376 |
return relations, kwargs |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
377 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
378 |
def add_entity(self, session, entity): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
379 |
"""add a new entity to the source""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
380 |
raise NotImplementedError() |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
381 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
382 |
def update_entity(self, session, entity): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
383 |
"""update an entity in the source""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
384 |
relations, kwargs = self._entity_relations_and_kwargs(session, entity) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
385 |
cu = session.pool[self.uri] |
5363
d336476dcf57
[3.8 api update] remaining cache keys
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5323
diff
changeset
|
386 |
cu.execute('SET %s WHERE X eid %%(x)s' % ','.join(relations), kwargs) |
1805 | 387 |
self._query_cache.clear() |
3043
7bcb7f1c0f46
force cache clear when external entity is edited, we don't know what may occurs in external repository hooks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3016
diff
changeset
|
388 |
entity.clear_all_caches() |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
389 |
|
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4816
diff
changeset
|
390 |
def delete_entity(self, session, entity): |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
391 |
"""delete an entity from the source""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
392 |
cu = session.pool[self.uri] |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4816
diff
changeset
|
393 |
cu.execute('DELETE %s X WHERE X eid %%(x)s' % entity.__regid__, |
5363
d336476dcf57
[3.8 api update] remaining cache keys
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5323
diff
changeset
|
394 |
{'x': self.eid2extid(entity.eid, session)}) |
1805 | 395 |
self._query_cache.clear() |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
396 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
397 |
def add_relation(self, session, subject, rtype, object): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
398 |
"""add a relation to the source""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
399 |
cu = session.pool[self.uri] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
400 |
cu.execute('SET X %s Y WHERE X eid %%(x)s, Y eid %%(y)s' % rtype, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
401 |
{'x': self.eid2extid(subject, session), |
5363
d336476dcf57
[3.8 api update] remaining cache keys
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5323
diff
changeset
|
402 |
'y': self.eid2extid(object, session)}) |
1805 | 403 |
self._query_cache.clear() |
3043
7bcb7f1c0f46
force cache clear when external entity is edited, we don't know what may occurs in external repository hooks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3016
diff
changeset
|
404 |
session.entity_from_eid(subject).clear_all_caches() |
7bcb7f1c0f46
force cache clear when external entity is edited, we don't know what may occurs in external repository hooks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3016
diff
changeset
|
405 |
session.entity_from_eid(object).clear_all_caches() |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
406 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
407 |
def delete_relation(self, session, subject, rtype, object): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
408 |
"""delete a relation from the source""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
409 |
cu = session.pool[self.uri] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
410 |
cu.execute('DELETE X %s Y WHERE X eid %%(x)s, Y eid %%(y)s' % rtype, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
411 |
{'x': self.eid2extid(subject, session), |
5363
d336476dcf57
[3.8 api update] remaining cache keys
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5323
diff
changeset
|
412 |
'y': self.eid2extid(object, session)}) |
1805 | 413 |
self._query_cache.clear() |
3043
7bcb7f1c0f46
force cache clear when external entity is edited, we don't know what may occurs in external repository hooks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3016
diff
changeset
|
414 |
session.entity_from_eid(subject).clear_all_caches() |
7bcb7f1c0f46
force cache clear when external entity is edited, we don't know what may occurs in external repository hooks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3016
diff
changeset
|
415 |
session.entity_from_eid(object).clear_all_caches() |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
416 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
417 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
418 |
class RQL2RQL(object): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
419 |
"""translate a local rql query to be executed on a distant repository""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
420 |
def __init__(self, source): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
421 |
self.source = source |
390
739d12586b9d
avoid potentialy mis-interpreted attribute error
sylvain.thenault@logilab.fr
parents:
257
diff
changeset
|
422 |
self.current_operator = None |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
423 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
424 |
def _accept_children(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
425 |
res = [] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
426 |
for child in node.children: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
427 |
rql = child.accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
428 |
if rql is not None: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
429 |
res.append(rql) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
430 |
return res |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
431 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
432 |
def generate(self, session, rqlst, args): |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
433 |
self._session = session |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
434 |
self.kwargs = args |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
435 |
self.need_translation = False |
5174
78438ad513ca
#759035: Automate addition of eid cachekey in RQL analysis
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
436 |
return self.visit_union(rqlst) |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
437 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
438 |
def visit_union(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
439 |
s = self._accept_children(node) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
440 |
if len(s) > 1: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
441 |
return ' UNION '.join('(%s)' % q for q in s) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
442 |
return s[0] |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
443 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
444 |
def visit_select(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
445 |
"""return the tree as an encoded rql string""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
446 |
self._varmaker = rqlvar_maker(defined=node.defined_vars.copy()) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
447 |
self._const_var = {} |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
448 |
if node.distinct: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
449 |
base = 'DISTINCT Any' |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
450 |
else: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
451 |
base = 'Any' |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
452 |
s = ['%s %s' % (base, ','.join(v.accept(self) for v in node.selection))] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
453 |
if node.groupby: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
454 |
s.append('GROUPBY %s' % ', '.join(group.accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
455 |
for group in node.groupby)) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
456 |
if node.orderby: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
457 |
s.append('ORDERBY %s' % ', '.join(self.visit_sortterm(term) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
458 |
for term in node.orderby)) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
459 |
if node.limit is not None: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
460 |
s.append('LIMIT %s' % node.limit) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
461 |
if node.offset: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
462 |
s.append('OFFSET %s' % node.offset) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
463 |
restrictions = [] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
464 |
if node.where is not None: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
465 |
nr = node.where.accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
466 |
if nr is not None: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
467 |
restrictions.append(nr) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
468 |
if restrictions: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
469 |
s.append('WHERE %s' % ','.join(restrictions)) |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
470 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
471 |
if node.having: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
472 |
s.append('HAVING %s' % ', '.join(term.accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
473 |
for term in node.having)) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
474 |
subqueries = [] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
475 |
for subquery in node.with_: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
476 |
subqueries.append('%s BEING (%s)' % (','.join(ca.name for ca in subquery.aliases), |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
477 |
self.visit_union(subquery.query))) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
478 |
if subqueries: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
479 |
s.append('WITH %s' % (','.join(subqueries))) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
480 |
return ' '.join(s) |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
481 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
482 |
def visit_and(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
483 |
res = self._accept_children(node) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
484 |
if res: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
485 |
return ', '.join(res) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
486 |
return |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
487 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
488 |
def visit_or(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
489 |
res = self._accept_children(node) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
490 |
if len(res) > 1: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
491 |
return ' OR '.join('(%s)' % rql for rql in res) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
492 |
elif res: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
493 |
return res[0] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
494 |
return |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
495 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
496 |
def visit_not(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
497 |
rql = node.children[0].accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
498 |
if rql: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
499 |
return 'NOT (%s)' % rql |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
500 |
return |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
501 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
502 |
def visit_exists(self, node): |
5582
3e133b29a1a4
[rql2sql] follow rql 0.26.1 changes: NOT nodes normalization, allowing simplification of sql generation, and fix #XXX
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5426
diff
changeset
|
503 |
rql = node.children[0].accept(self) |
3e133b29a1a4
[rql2sql] follow rql 0.26.1 changes: NOT nodes normalization, allowing simplification of sql generation, and fix #XXX
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5426
diff
changeset
|
504 |
if rql: |
3e133b29a1a4
[rql2sql] follow rql 0.26.1 changes: NOT nodes normalization, allowing simplification of sql generation, and fix #XXX
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5426
diff
changeset
|
505 |
return 'EXISTS(%s)' % rql |
3e133b29a1a4
[rql2sql] follow rql 0.26.1 changes: NOT nodes normalization, allowing simplification of sql generation, and fix #XXX
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5426
diff
changeset
|
506 |
return |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
507 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
508 |
def visit_relation(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
509 |
try: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
510 |
if isinstance(node.children[0], Constant): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
511 |
# simplified rqlst, reintroduce eid relation |
391
42693fe3ef6f
skip neged relation here as well
sylvain.thenault@logilab.fr
parents:
390
diff
changeset
|
512 |
try: |
42693fe3ef6f
skip neged relation here as well
sylvain.thenault@logilab.fr
parents:
390
diff
changeset
|
513 |
restr, lhs = self.process_eid_const(node.children[0]) |
42693fe3ef6f
skip neged relation here as well
sylvain.thenault@logilab.fr
parents:
390
diff
changeset
|
514 |
except UnknownEid: |
42693fe3ef6f
skip neged relation here as well
sylvain.thenault@logilab.fr
parents:
390
diff
changeset
|
515 |
# can safely skip not relation with an unsupported eid |
5582
3e133b29a1a4
[rql2sql] follow rql 0.26.1 changes: NOT nodes normalization, allowing simplification of sql generation, and fix #XXX
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5426
diff
changeset
|
516 |
if neged_relation(node): |
391
42693fe3ef6f
skip neged relation here as well
sylvain.thenault@logilab.fr
parents:
390
diff
changeset
|
517 |
return |
42693fe3ef6f
skip neged relation here as well
sylvain.thenault@logilab.fr
parents:
390
diff
changeset
|
518 |
raise |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
519 |
else: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
520 |
lhs = node.children[0].accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
521 |
restr = None |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
522 |
except UnknownEid: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
523 |
# can safely skip not relation with an unsupported eid |
5582
3e133b29a1a4
[rql2sql] follow rql 0.26.1 changes: NOT nodes normalization, allowing simplification of sql generation, and fix #XXX
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5426
diff
changeset
|
524 |
if neged_relation(node): |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
525 |
return |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
526 |
# XXX what about optional relation or outer NOT EXISTS() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
527 |
raise |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
528 |
if node.optional in ('left', 'both'): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
529 |
lhs += '?' |
3689
deb13e88e037
follow yams 0.25 api changes to improve performance
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3605
diff
changeset
|
530 |
if node.r_type == 'eid' or not self.source.schema.rschema(node.r_type).final: |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
531 |
self.need_translation = True |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
532 |
self.current_operator = node.operator() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
533 |
if isinstance(node.children[0], Constant): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
534 |
self.current_etypes = (node.children[0].uidtype,) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
535 |
else: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
536 |
self.current_etypes = node.children[0].variable.stinfo['possibletypes'] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
537 |
try: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
538 |
rhs = node.children[1].accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
539 |
except UnknownEid: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
540 |
# can safely skip not relation with an unsupported eid |
5582
3e133b29a1a4
[rql2sql] follow rql 0.26.1 changes: NOT nodes normalization, allowing simplification of sql generation, and fix #XXX
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5426
diff
changeset
|
541 |
if neged_relation(node): |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
542 |
return |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
543 |
# XXX what about optional relation or outer NOT EXISTS() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
544 |
raise |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
545 |
except ReplaceByInOperator, ex: |
1952
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1805
diff
changeset
|
546 |
rhs = 'IN (%s)' % ','.join(eid for eid in ex.eids) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
547 |
self.need_translation = False |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
548 |
self.current_operator = None |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
549 |
if node.optional in ('right', 'both'): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
550 |
rhs += '?' |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
551 |
if restr is not None: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
552 |
return '%s %s %s, %s' % (lhs, node.r_type, rhs, restr) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
553 |
return '%s %s %s' % (lhs, node.r_type, rhs) |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
554 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
555 |
def visit_comparison(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
556 |
if node.operator in ('=', 'IS'): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
557 |
return node.children[0].accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
558 |
return '%s %s' % (node.operator.encode(), |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
559 |
node.children[0].accept(self)) |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
560 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
561 |
def visit_mathexpression(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
562 |
return '(%s %s %s)' % (node.children[0].accept(self), |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
563 |
node.operator.encode(), |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
564 |
node.children[1].accept(self)) |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
565 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
566 |
def visit_function(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
567 |
#if node.name == 'IN': |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
568 |
res = [] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
569 |
for child in node.children: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
570 |
try: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
571 |
rql = child.accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
572 |
except UnknownEid, ex: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
573 |
continue |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
574 |
res.append(rql) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
575 |
if not res: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
576 |
raise ex |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
577 |
return '%s(%s)' % (node.name, ', '.join(res)) |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
578 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
579 |
def visit_constant(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
580 |
if self.need_translation or node.uidtype: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
581 |
if node.type == 'Int': |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
582 |
return str(self.eid2extid(node.value)) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
583 |
if node.type == 'Substitute': |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
584 |
key = node.value |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
585 |
# ensure we have not yet translated the value... |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
586 |
if not key in self._const_var: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
587 |
self.kwargs[key] = self.eid2extid(self.kwargs[key]) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
588 |
self._const_var[key] = None |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
589 |
return node.as_string() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
590 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
591 |
def visit_variableref(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
592 |
"""get the sql name for a variable reference""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
593 |
return node.name |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
594 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
595 |
def visit_sortterm(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
596 |
if node.asc: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
597 |
return node.term.accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
598 |
return '%s DESC' % node.term.accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
599 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
600 |
def process_eid_const(self, const): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
601 |
value = const.eval(self.kwargs) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
602 |
try: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
603 |
return None, self._const_var[value] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
604 |
except: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
605 |
var = self._varmaker.next() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
606 |
self.need_translation = True |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
607 |
restr = '%s eid %s' % (var, self.visit_constant(const)) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
608 |
self.need_translation = False |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
609 |
self._const_var[value] = var |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
610 |
return restr, var |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
611 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
612 |
def eid2extid(self, eid): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
613 |
try: |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
614 |
return self.source.eid2extid(eid, self._session) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
615 |
except UnknownEid: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
616 |
operator = self.current_operator |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
617 |
if operator is not None and operator != '=': |
1954 | 618 |
# deal with query like "X eid > 12" |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
619 |
# |
1954 | 620 |
# The problem is that eid order in the external source may |
621 |
# differ from the local source |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
622 |
# |
1954 | 623 |
# So search for all eids from this source matching the condition |
624 |
# locally and then to replace the "> 12" branch by "IN (eids)" |
|
625 |
# |
|
626 |
# XXX we may have to insert a huge number of eids...) |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
627 |
sql = "SELECT extid FROM entities WHERE source='%s' AND type IN (%s) AND eid%s%s" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
628 |
etypes = ','.join("'%s'" % etype for etype in self.current_etypes) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
629 |
cu = self._session.system_sql(sql % (self.source.uri, etypes, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
630 |
operator, eid)) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
631 |
# XXX buggy cu.rowcount which may be zero while there are some |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
632 |
# results |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
633 |
rows = cu.fetchall() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
634 |
if rows: |
1952
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1805
diff
changeset
|
635 |
raise ReplaceByInOperator((b64decode(r[0]) for r in rows)) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
636 |
raise |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
637 |