author | Sylvain Thénault <sylvain.thenault@logilab.fr> |
Mon, 17 Oct 2011 15:08:25 +0200 | |
changeset 7963 | 059914a691a9 |
parent 7895 | 0a967180794b |
child 8244 | c7d89541e3c5 |
permissions | -rw-r--r-- |
6944
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
1 |
# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved. |
5421
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
2 |
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
3 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
4 |
# This file is part of CubicWeb. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
5 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
6 |
# CubicWeb is free software: you can redistribute it and/or modify it under the |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
7 |
# terms of the GNU Lesser General Public License as published by the Free |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
8 |
# Software Foundation, either version 2.1 of the License, or (at your option) |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
9 |
# any later version. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
10 |
# |
5424
8ecbcbff9777
replace logilab-common by CubicWeb in disclaimer
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5421
diff
changeset
|
11 |
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT |
5421
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
12 |
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
13 |
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
14 |
# details. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
15 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
16 |
# You should have received a copy of the GNU Lesser General Public License along |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
17 |
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>. |
6128 | 18 |
"""Source to query another RQL repository using pyro""" |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
19 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
20 |
__docformat__ = "restructuredtext en" |
6582
8eb7883b4223
[pylint] fix a bug of pylint detected errors and i18n pb (calling builtins._ instead of req._)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6427
diff
changeset
|
21 |
_ = unicode |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
22 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
23 |
import threading |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
24 |
from os.path import join |
1016
26387b836099
use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
25 |
from time import mktime |
26387b836099
use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
26 |
from datetime import datetime |
1952
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1805
diff
changeset
|
27 |
from base64 import b64decode |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
28 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
29 |
from Pyro.errors import PyroError, ConnectionClosedError |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
30 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
31 |
from logilab.common.configuration import REQUIRED |
6625
54847e2c4a50
[pyro source] new option to control wether external entities should be skipped or not
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6582
diff
changeset
|
32 |
from logilab.common.optik_ext import check_yn |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
33 |
|
6944
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
34 |
from yams.schema import role_name |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
35 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
36 |
from rql.nodes import Constant |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
37 |
from rql.utils import rqlvar_maker |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
38 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
39 |
from cubicweb import dbapi, server |
6944
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
40 |
from cubicweb import ValidationError, BadConnectionId, UnknownEid, ConnectionError |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
41 |
from cubicweb.schema import VIRTUAL_RTYPES |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
42 |
from cubicweb.cwconfig import register_persistent_options |
2625
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2593
diff
changeset
|
43 |
from cubicweb.server.sources import (AbstractSource, ConnectionWrapper, |
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2593
diff
changeset
|
44 |
TimedCache, dbg_st_search, dbg_results) |
5582
3e133b29a1a4
[rql2sql] follow rql 0.26.1 changes: NOT nodes normalization, allowing simplification of sql generation, and fix #XXX
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5426
diff
changeset
|
45 |
from cubicweb.server.msplanner import neged_relation |
3016
5787d1cc8106
[rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2763
diff
changeset
|
46 |
|
5787d1cc8106
[rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2763
diff
changeset
|
47 |
def uidtype(union, col, etype, args): |
5787d1cc8106
[rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2763
diff
changeset
|
48 |
select, col = union.locate_subquery(col, etype, args) |
5787d1cc8106
[rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2763
diff
changeset
|
49 |
return getattr(select.selection[col], 'uidtype', None) |
5787d1cc8106
[rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2763
diff
changeset
|
50 |
|
5787d1cc8106
[rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2763
diff
changeset
|
51 |
|
1138
22f634977c95
make pylint happy, fix some bugs on the way
sylvain.thenault@logilab.fr
parents:
1016
diff
changeset
|
52 |
class ReplaceByInOperator(Exception): |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
53 |
def __init__(self, eids): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
54 |
self.eids = eids |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
55 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
56 |
class PyroRQLSource(AbstractSource): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
57 |
"""External repository source, using Pyro connection""" |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
58 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
59 |
# boolean telling if modification hooks should be called when something is |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
60 |
# modified in this source |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
61 |
should_call_hooks = False |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
62 |
# boolean telling if the repository should connect to this source during |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
63 |
# migration |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
64 |
connect_for_migration = False |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
65 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
66 |
options = ( |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
67 |
# XXX pyro-ns host/port |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
68 |
('pyro-ns-id', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
69 |
{'type' : 'string', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
70 |
'default': REQUIRED, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
71 |
'help': 'identifier of the repository in the pyro name server', |
5323
329b4f6d18b4
[config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5174
diff
changeset
|
72 |
'group': 'pyro-source', 'level': 0, |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
73 |
}), |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
74 |
('cubicweb-user', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
75 |
{'type' : 'string', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
76 |
'default': REQUIRED, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
77 |
'help': 'user to use for connection on the distant repository', |
5323
329b4f6d18b4
[config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5174
diff
changeset
|
78 |
'group': 'pyro-source', 'level': 0, |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
79 |
}), |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
80 |
('cubicweb-password', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
81 |
{'type' : 'password', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
82 |
'default': '', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
83 |
'help': 'user to use for connection on the distant repository', |
5323
329b4f6d18b4
[config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5174
diff
changeset
|
84 |
'group': 'pyro-source', 'level': 0, |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
85 |
}), |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
86 |
('base-url', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
87 |
{'type' : 'string', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
88 |
'default': '', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
89 |
'help': 'url of the web site for the distant repository, if you want ' |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
90 |
'to generate external link to entities from this repository', |
5323
329b4f6d18b4
[config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5174
diff
changeset
|
91 |
'group': 'pyro-source', 'level': 1, |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
92 |
}), |
6625
54847e2c4a50
[pyro source] new option to control wether external entities should be skipped or not
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6582
diff
changeset
|
93 |
('skip-external-entities', |
54847e2c4a50
[pyro source] new option to control wether external entities should be skipped or not
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6582
diff
changeset
|
94 |
{'type' : 'yn', |
54847e2c4a50
[pyro source] new option to control wether external entities should be skipped or not
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6582
diff
changeset
|
95 |
'default': False, |
54847e2c4a50
[pyro source] new option to control wether external entities should be skipped or not
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6582
diff
changeset
|
96 |
'help': 'should entities not local to the source be considered or not', |
54847e2c4a50
[pyro source] new option to control wether external entities should be skipped or not
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6582
diff
changeset
|
97 |
'group': 'pyro-source', 'level': 0, |
54847e2c4a50
[pyro source] new option to control wether external entities should be skipped or not
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6582
diff
changeset
|
98 |
}), |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
99 |
('pyro-ns-host', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
100 |
{'type' : 'string', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
101 |
'default': None, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
102 |
'help': 'Pyro name server\'s host. If not set, default to the value \ |
2665
0c6281487f90
[pyro] use lgc.pyro_ext, simplify pyro related options
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2625
diff
changeset
|
103 |
from all_in_one.conf. It may contains port information using <host>:<port> notation.', |
5323
329b4f6d18b4
[config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5174
diff
changeset
|
104 |
'group': 'pyro-source', 'level': 1, |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
105 |
}), |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
106 |
('pyro-ns-group', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
107 |
{'type' : 'string', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
108 |
'default': None, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
109 |
'help': 'Pyro name server\'s group where the repository will be \ |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
110 |
registered. If not set, default to the value from all_in_one.conf.', |
6632 | 111 |
'group': 'pyro-source', 'level': 2, |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
112 |
}), |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
113 |
('synchronization-interval', |
6946
e350771c23a3
[pyrorql source] using 'time' option type is more convenient
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6945
diff
changeset
|
114 |
{'type' : 'time', |
e350771c23a3
[pyrorql source] using 'time' option type is more convenient
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6945
diff
changeset
|
115 |
'default': '5min', |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
116 |
'help': 'interval between synchronization with the external \ |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
117 |
repository (default to 5 minutes).', |
5323
329b4f6d18b4
[config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5174
diff
changeset
|
118 |
'group': 'pyro-source', 'level': 2, |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
119 |
}), |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
120 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
121 |
) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
122 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
123 |
PUBLIC_KEYS = AbstractSource.PUBLIC_KEYS + ('base-url',) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
124 |
_conn = None |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
125 |
|
6945
28bf94d062a9
[sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6944
diff
changeset
|
126 |
def __init__(self, repo, source_config, eid=None): |
28bf94d062a9
[sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6944
diff
changeset
|
127 |
AbstractSource.__init__(self, repo, source_config, eid) |
28bf94d062a9
[sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6944
diff
changeset
|
128 |
self.update_config(None, self.check_conf_dict(eid, source_config, |
28bf94d062a9
[sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6944
diff
changeset
|
129 |
fail_if_unknown=False)) |
5642
6a90357b9769
TimedCache now only accepts values expressed in seconds
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
5426
diff
changeset
|
130 |
self._query_cache = TimedCache(1800) |
6945
28bf94d062a9
[sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6944
diff
changeset
|
131 |
|
28bf94d062a9
[sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6944
diff
changeset
|
132 |
def update_config(self, source_entity, processed_config): |
28bf94d062a9
[sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6944
diff
changeset
|
133 |
"""update configuration from source entity""" |
28bf94d062a9
[sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6944
diff
changeset
|
134 |
# XXX get it through pyro if unset |
28bf94d062a9
[sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6944
diff
changeset
|
135 |
baseurl = processed_config.get('base-url') |
28bf94d062a9
[sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6944
diff
changeset
|
136 |
if baseurl and not baseurl.endswith('/'): |
28bf94d062a9
[sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6944
diff
changeset
|
137 |
processed_config['base-url'] += '/' |
28bf94d062a9
[sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6944
diff
changeset
|
138 |
self.config = processed_config |
28bf94d062a9
[sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6944
diff
changeset
|
139 |
self._skip_externals = processed_config['skip-external-entities'] |
6958
861251f125cf
[pyro source] benefit from addition of latest_retrieval on CWSource
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6957
diff
changeset
|
140 |
if source_entity is not None: |
861251f125cf
[pyro source] benefit from addition of latest_retrieval on CWSource
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6957
diff
changeset
|
141 |
self.latest_retrieval = source_entity.latest_retrieval |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
142 |
|
2763
39b42e158249
[ms] proper reset cache on external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2665
diff
changeset
|
143 |
def reset_caches(self): |
39b42e158249
[ms] proper reset cache on external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2665
diff
changeset
|
144 |
"""method called during test to reset potential source caches""" |
5642
6a90357b9769
TimedCache now only accepts values expressed in seconds
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
5426
diff
changeset
|
145 |
self._query_cache = TimedCache(1800) |
2763
39b42e158249
[ms] proper reset cache on external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2665
diff
changeset
|
146 |
|
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6946
diff
changeset
|
147 |
def init(self, activated, source_entity): |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
148 |
"""method called by the repository once ready to handle request""" |
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6946
diff
changeset
|
149 |
self.load_mapping(source_entity._cw) |
6724
24bf6f181d0e
[pyro source] store pyro source mapping file into the database
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6672
diff
changeset
|
150 |
if activated: |
6946
e350771c23a3
[pyrorql source] using 'time' option type is more convenient
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6945
diff
changeset
|
151 |
interval = self.config['synchronization-interval'] |
6724
24bf6f181d0e
[pyro source] store pyro source mapping file into the database
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6672
diff
changeset
|
152 |
self.repo.looping_task(interval, self.synchronize) |
24bf6f181d0e
[pyro source] store pyro source mapping file into the database
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6672
diff
changeset
|
153 |
self.repo.looping_task(self._query_cache.ttl.seconds/10, |
24bf6f181d0e
[pyro source] store pyro source mapping file into the database
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6672
diff
changeset
|
154 |
self._query_cache.clear_expired) |
6958
861251f125cf
[pyro source] benefit from addition of latest_retrieval on CWSource
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6957
diff
changeset
|
155 |
self.latest_retrieval = source_entity.latest_retrieval |
6724
24bf6f181d0e
[pyro source] store pyro source mapping file into the database
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6672
diff
changeset
|
156 |
|
24bf6f181d0e
[pyro source] store pyro source mapping file into the database
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6672
diff
changeset
|
157 |
def load_mapping(self, session=None): |
24bf6f181d0e
[pyro source] store pyro source mapping file into the database
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6672
diff
changeset
|
158 |
self.support_entities = {} |
24bf6f181d0e
[pyro source] store pyro source mapping file into the database
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6672
diff
changeset
|
159 |
self.support_relations = {} |
24bf6f181d0e
[pyro source] store pyro source mapping file into the database
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6672
diff
changeset
|
160 |
self.dont_cross_relations = set(('owned_by', 'created_by')) |
24bf6f181d0e
[pyro source] store pyro source mapping file into the database
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6672
diff
changeset
|
161 |
self.cross_relations = set() |
24bf6f181d0e
[pyro source] store pyro source mapping file into the database
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6672
diff
changeset
|
162 |
assert self.eid is not None |
6944
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
163 |
self._schemacfg_idx = {} |
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6946
diff
changeset
|
164 |
self._load_mapping(session) |
6944
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
165 |
|
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
166 |
etype_options = set(('write',)) |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
167 |
rtype_options = set(('maycross', 'dontcross', 'write',)) |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
168 |
|
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
169 |
def _check_options(self, schemacfg, allowedoptions): |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
170 |
if schemacfg.options: |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
171 |
options = set(w.strip() for w in schemacfg.options.split(':')) |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
172 |
else: |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
173 |
options = set() |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
174 |
if options - allowedoptions: |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
175 |
options = ', '.join(sorted(options - allowedoptions)) |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
176 |
msg = _('unknown option(s): %s' % options) |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
177 |
raise ValidationError(schemacfg.eid, {role_name('options', 'subject'): msg}) |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
178 |
return options |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
179 |
|
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
180 |
def add_schema_config(self, schemacfg, checkonly=False): |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
181 |
"""added CWSourceSchemaConfig, modify mapping accordingly""" |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
182 |
try: |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
183 |
ertype = schemacfg.schema.name |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
184 |
except AttributeError: |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
185 |
msg = schemacfg._cw._("attribute/relation can't be mapped, only " |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
186 |
"entity and relation types") |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
187 |
raise ValidationError(schemacfg.eid, {role_name('cw_for_schema', 'subject'): msg}) |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
188 |
if schemacfg.schema.__regid__ == 'CWEType': |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
189 |
options = self._check_options(schemacfg, self.etype_options) |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
190 |
if not checkonly: |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
191 |
self.support_entities[ertype] = 'write' in options |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
192 |
else: # CWRType |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
193 |
if ertype in ('is', 'is_instance_of', 'cw_source') or ertype in VIRTUAL_RTYPES: |
7879
9aae456abab5
[pylint] fix pylint detected errors and tweak it so that pylint -E will be much less verbose next time (+ update some copyrights on the way)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7815
diff
changeset
|
194 |
msg = schemacfg._cw._('%s relation should not be in mapped') % ertype |
6944
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
195 |
raise ValidationError(schemacfg.eid, {role_name('cw_for_schema', 'subject'): msg}) |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
196 |
options = self._check_options(schemacfg, self.rtype_options) |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
197 |
if 'dontcross' in options: |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
198 |
if 'maycross' in options: |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
199 |
msg = schemacfg._("can't mix dontcross and maycross options") |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
200 |
raise ValidationError(schemacfg.eid, {role_name('options', 'subject'): msg}) |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
201 |
if 'write' in options: |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
202 |
msg = schemacfg._("can't mix dontcross and write options") |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
203 |
raise ValidationError(schemacfg.eid, {role_name('options', 'subject'): msg}) |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
204 |
if not checkonly: |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
205 |
self.dont_cross_relations.add(ertype) |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
206 |
elif not checkonly: |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
207 |
self.support_relations[ertype] = 'write' in options |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
208 |
if 'maycross' in options: |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
209 |
self.cross_relations.add(ertype) |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
210 |
if not checkonly: |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
211 |
# add to an index to ease deletion handling |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
212 |
self._schemacfg_idx[schemacfg.eid] = ertype |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
213 |
|
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
214 |
def del_schema_config(self, schemacfg, checkonly=False): |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
215 |
"""deleted CWSourceSchemaConfig, modify mapping accordingly""" |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
216 |
if checkonly: |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
217 |
return |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
218 |
try: |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
219 |
ertype = self._schemacfg_idx[schemacfg.eid] |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
220 |
if ertype[0].isupper(): |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
221 |
del self.support_entities[ertype] |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
222 |
else: |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
223 |
if ertype in self.support_relations: |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
224 |
del self.support_relations[ertype] |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
225 |
if ertype in self.cross_relations: |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
226 |
self.cross_relations.remove(ertype) |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
227 |
else: |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
228 |
self.dont_cross_relations.remove(ertype) |
7815
2a164a9cf81c
[exceptions] stop catching any exception in various places (closes #1942716)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7697
diff
changeset
|
229 |
except Exception: |
6944
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
230 |
self.error('while updating mapping consequently to removal of %s', |
0cf10429ad39
[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6941
diff
changeset
|
231 |
schemacfg) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
232 |
|
6655
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
233 |
def local_eid(self, cnx, extid, session): |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
234 |
etype, dexturi, dextid = cnx.describe(extid) |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
235 |
if dexturi == 'system' or not ( |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
236 |
dexturi in self.repo.sources_by_uri or self._skip_externals): |
7526
ae31063f3274
cleanup and assert we don't import unexpected stuff
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7499
diff
changeset
|
237 |
assert etype in self.support_entities, etype |
7399
972ed1843bd8
[multi-sources] support for moving an entity from an external source (closes #343818)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7398
diff
changeset
|
238 |
eid = self.repo.extid2eid(self, str(extid), etype, session) |
972ed1843bd8
[multi-sources] support for moving an entity from an external source (closes #343818)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7398
diff
changeset
|
239 |
if eid > 0: |
972ed1843bd8
[multi-sources] support for moving an entity from an external source (closes #343818)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7398
diff
changeset
|
240 |
return eid, True |
972ed1843bd8
[multi-sources] support for moving an entity from an external source (closes #343818)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7398
diff
changeset
|
241 |
elif dexturi in self.repo.sources_by_uri: |
6655
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
242 |
source = self.repo.sources_by_uri[dexturi] |
7398
26695dd703d8
[repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7293
diff
changeset
|
243 |
cnx = session.cnxset.connection(source.uri) |
6655
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
244 |
eid = source.local_eid(cnx, dextid, session)[0] |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
245 |
return eid, False |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
246 |
return None, None |
6625
54847e2c4a50
[pyro source] new option to control wether external entities should be skipped or not
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6582
diff
changeset
|
247 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
248 |
def synchronize(self, mtime=None): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
249 |
"""synchronize content known by this repository with content in the |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
250 |
external repository |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
251 |
""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
252 |
self.info('synchronizing pyro source %s', self.uri) |
519
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
253 |
cnx = self.get_connection() |
2271
5483155a1e17
avoid additional tb in logs when we can't connect to the external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2269
diff
changeset
|
254 |
try: |
5483155a1e17
avoid additional tb in logs when we can't connect to the external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2269
diff
changeset
|
255 |
extrepo = cnx._repo |
5483155a1e17
avoid additional tb in logs when we can't connect to the external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2269
diff
changeset
|
256 |
except AttributeError: |
5483155a1e17
avoid additional tb in logs when we can't connect to the external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2269
diff
changeset
|
257 |
# fake connection wrapper returned when we can't connect to the |
5483155a1e17
avoid additional tb in logs when we can't connect to the external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2269
diff
changeset
|
258 |
# external source (hence we've no chance to synchronize...) |
5483155a1e17
avoid additional tb in logs when we can't connect to the external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2269
diff
changeset
|
259 |
return |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
260 |
etypes = self.support_entities.keys() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
261 |
if mtime is None: |
6958
861251f125cf
[pyro source] benefit from addition of latest_retrieval on CWSource
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6957
diff
changeset
|
262 |
mtime = self.latest_retrieval |
6724
24bf6f181d0e
[pyro source] store pyro source mapping file into the database
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6672
diff
changeset
|
263 |
updatetime, modified, deleted = extrepo.entities_modified_since( |
24bf6f181d0e
[pyro source] store pyro source mapping file into the database
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6672
diff
changeset
|
264 |
etypes, mtime) |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
265 |
self._query_cache.clear() |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
266 |
repo = self.repo |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
267 |
session = repo.internal_session() |
4816
c02583cb80a9
repair stuff broken by fti handling changes
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4212
diff
changeset
|
268 |
source = repo.system_source |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
269 |
try: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
270 |
for etype, extid in modified: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
271 |
try: |
6672
2008fd2f628c
[ms] fix error in synchronisation: local_eid return a 2-uple
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6655
diff
changeset
|
272 |
eid = self.local_eid(cnx, extid, session)[0] |
6655
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
273 |
if eid is not None: |
519
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
274 |
rset = session.eid_rset(eid, etype) |
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
275 |
entity = rset.get_entity(0, 0) |
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
276 |
entity.complete(entity.e_schema.indexable_attributes()) |
4816
c02583cb80a9
repair stuff broken by fti handling changes
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4212
diff
changeset
|
277 |
source.index_entity(session, entity) |
7815
2a164a9cf81c
[exceptions] stop catching any exception in various places (closes #1942716)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7697
diff
changeset
|
278 |
except Exception: |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
279 |
self.exception('while updating %s with external id %s of source %s', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
280 |
etype, extid, self.uri) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
281 |
continue |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
282 |
for etype, extid in deleted: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
283 |
try: |
7697
ef50074a0314
[repo api] deprecates source.extid2eid, only the system source should implement it,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7650
diff
changeset
|
284 |
eid = self.repo.extid2eid(self, str(extid), etype, session, |
ef50074a0314
[repo api] deprecates source.extid2eid, only the system source should implement it,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7650
diff
changeset
|
285 |
insert=False) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
286 |
# entity has been deleted from external repository but is not known here |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
287 |
if eid is not None: |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4816
diff
changeset
|
288 |
entity = session.entity_from_eid(eid, etype) |
7884
35d2e2f4e10a
[repo] cleanup [_]delete_info prototype: extid argument isn't used
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7879
diff
changeset
|
289 |
repo.delete_info(session, entity, self.uri, |
6941
9ed02daa7dbb
[repo, ms] optimise external source deletion by using source's eid instead of name
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6724
diff
changeset
|
290 |
scleanup=self.eid) |
7815
2a164a9cf81c
[exceptions] stop catching any exception in various places (closes #1942716)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7697
diff
changeset
|
291 |
except Exception: |
7885
9454b7ef5ae4
[ms, test] propagate exception in test mode
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7884
diff
changeset
|
292 |
if self.repo.config.mode == 'test': |
9454b7ef5ae4
[ms, test] propagate exception in test mode
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7884
diff
changeset
|
293 |
raise |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
294 |
self.exception('while updating %s with external id %s of source %s', |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
295 |
etype, extid, self.uri) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
296 |
continue |
6958
861251f125cf
[pyro source] benefit from addition of latest_retrieval on CWSource
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6957
diff
changeset
|
297 |
self.latest_retrieval = updatetime |
861251f125cf
[pyro source] benefit from addition of latest_retrieval on CWSource
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6957
diff
changeset
|
298 |
session.execute('SET X latest_retrieval %(date)s WHERE X eid %(x)s', |
861251f125cf
[pyro source] benefit from addition of latest_retrieval on CWSource
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6957
diff
changeset
|
299 |
{'x': self.eid, 'date': self.latest_retrieval}) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
300 |
session.commit() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
301 |
finally: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
302 |
session.close() |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
303 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
304 |
def _get_connection(self): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
305 |
"""open and return a connection to the source""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
306 |
nshost = self.config.get('pyro-ns-host') or self.repo.config['pyro-ns-host'] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
307 |
nsgroup = self.config.get('pyro-ns-group') or self.repo.config['pyro-ns-group'] |
3527
efeb16ff93f3
log when connecting to an external pyro source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3043
diff
changeset
|
308 |
self.info('connecting to instance :%s.%s for user %s', |
3605
1466323ddb9c
fix debug msg
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3527
diff
changeset
|
309 |
nsgroup, self.config['pyro-ns-id'], self.config['cubicweb-user']) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
310 |
#cnxprops = ConnectionProperties(cnxtype=self.config['cnx-type']) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
311 |
return dbapi.connect(database=self.config['pyro-ns-id'], |
2269
0d1b7206fdc9
renamed arg
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1977
diff
changeset
|
312 |
login=self.config['cubicweb-user'], |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
313 |
password=self.config['cubicweb-password'], |
2665
0c6281487f90
[pyro] use lgc.pyro_ext, simplify pyro related options
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2625
diff
changeset
|
314 |
host=nshost, group=nsgroup, |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
315 |
setvreg=False) #cnxprops=cnxprops) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
316 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
317 |
def get_connection(self): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
318 |
try: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
319 |
return self._get_connection() |
7499
96412cfc28e2
[pyrorql] don't display traceback on connection error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6958
diff
changeset
|
320 |
except (ConnectionError, PyroError), ex: |
96412cfc28e2
[pyrorql] don't display traceback on connection error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6958
diff
changeset
|
321 |
self.critical("can't get connection to source %s: %s", self.uri, ex) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
322 |
return ConnectionWrapper() |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
323 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
324 |
def check_connection(self, cnx): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
325 |
"""check connection validity, return None if the connection is still valid |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
326 |
else a new connection |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
327 |
""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
328 |
# we have to transfer manually thread ownership. This can be done safely |
7398
26695dd703d8
[repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7293
diff
changeset
|
329 |
# since the connections set holding the connection is affected to one |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
330 |
# session/thread and can't be called simultaneously |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
331 |
try: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
332 |
cnx._repo._transferThread(threading.currentThread()) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
333 |
except AttributeError: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
334 |
# inmemory connection |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
335 |
pass |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
336 |
if not isinstance(cnx, ConnectionWrapper): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
337 |
try: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
338 |
cnx.check() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
339 |
return # ok |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
340 |
except (BadConnectionId, ConnectionClosedError): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
341 |
pass |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
342 |
# try to reconnect |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
343 |
return self.get_connection() |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
344 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
345 |
def syntax_tree_search(self, session, union, args=None, cachekey=None, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
346 |
varmap=None): |
2625
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2593
diff
changeset
|
347 |
assert dbg_st_search(self.uri, union, varmap, args, cachekey) |
1238
fa29b5b60107
set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
348 |
rqlkey = union.as_string(kwargs=args) |
fa29b5b60107
set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
349 |
try: |
fa29b5b60107
set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
350 |
results = self._query_cache[rqlkey] |
fa29b5b60107
set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
351 |
except KeyError: |
fa29b5b60107
set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
352 |
results = self._syntax_tree_search(session, union, args) |
fa29b5b60107
set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
353 |
self._query_cache[rqlkey] = results |
2625
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2593
diff
changeset
|
354 |
assert dbg_results(results) |
1238
fa29b5b60107
set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
355 |
return results |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
356 |
|
1238
fa29b5b60107
set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents:
520
diff
changeset
|
357 |
def _syntax_tree_search(self, session, union, args): |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
358 |
"""return result from this source for a rql query (actually from a rql |
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
359 |
syntax tree and a solution dictionary mapping each used variable to a |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
360 |
possible type). If cachekey is given, the query necessary to fetch the |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
361 |
results (but not the results themselves) may be cached using this key. |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
362 |
""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
363 |
if not args is None: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
364 |
args = args.copy() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
365 |
# get cached cursor anyway |
7398
26695dd703d8
[repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7293
diff
changeset
|
366 |
cu = session.cnxset[self.uri] |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
367 |
if cu is None: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
368 |
# this is a ConnectionWrapper instance |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
369 |
msg = session._("can't connect to source %s, some data may be missing") |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
370 |
session.set_shared_data('sources_error', msg % self.uri) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
371 |
return [] |
6655
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
372 |
translator = RQL2RQL(self) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
373 |
try: |
6655
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
374 |
rql = translator.generate(session, union, args) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
375 |
except UnknownEid, ex: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
376 |
if server.DEBUG: |
2625
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2593
diff
changeset
|
377 |
print ' unknown eid', ex, 'no results' |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
378 |
return [] |
2593
16d9419a4a79
F: start to handle binary debug log level on the server side
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2271
diff
changeset
|
379 |
if server.DEBUG & server.DBG_RQL: |
2625
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2593
diff
changeset
|
380 |
print ' translated rql', rql |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
381 |
try: |
5174
78438ad513ca
#759035: Automate addition of eid cachekey in RQL analysis
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
382 |
rset = cu.execute(rql, args) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
383 |
except Exception, ex: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
384 |
self.exception(str(ex)) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
385 |
msg = session._("error while querying source %s, some data may be missing") |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
386 |
session.set_shared_data('sources_error', msg % self.uri) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
387 |
return [] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
388 |
descr = rset.description |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
389 |
if rset: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
390 |
needtranslation = [] |
519
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
391 |
rows = rset.rows |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
392 |
for i, etype in enumerate(descr[0]): |
3689
deb13e88e037
follow yams 0.25 api changes to improve performance
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3605
diff
changeset
|
393 |
if (etype is None or not self.schema.eschema(etype).final |
3016
5787d1cc8106
[rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2763
diff
changeset
|
394 |
or uidtype(union, i, etype, args)): |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
395 |
needtranslation.append(i) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
396 |
if needtranslation: |
7398
26695dd703d8
[repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7293
diff
changeset
|
397 |
cnx = session.cnxset.connection(self.uri) |
519
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
398 |
for rowindex in xrange(rset.rowcount - 1, -1, -1): |
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
399 |
row = rows[rowindex] |
6655
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
400 |
localrow = False |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
401 |
for colindex in needtranslation: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
402 |
if row[colindex] is not None: # optional variable |
6655
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
403 |
eid, local = self.local_eid(cnx, row[colindex], session) |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
404 |
if local: |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
405 |
localrow = True |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
406 |
if eid is not None: |
519
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
407 |
row[colindex] = eid |
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
408 |
else: |
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
409 |
# skip this row |
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
410 |
del rows[rowindex] |
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
411 |
del descr[rowindex] |
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
412 |
break |
6655
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
413 |
else: |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
414 |
# skip row if it only contains eids of entities which |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
415 |
# are actually from a source we also know locally, |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
416 |
# except if some args specified (XXX should actually |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
417 |
# check if there are some args local to the source) |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
418 |
if not (translator.has_local_eid or localrow): |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
419 |
del rows[rowindex] |
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
420 |
del descr[rowindex] |
519
06390418cd9a
pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents:
417
diff
changeset
|
421 |
results = rows |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
422 |
else: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
423 |
results = [] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
424 |
return results |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
425 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
426 |
def _entity_relations_and_kwargs(self, session, entity): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
427 |
relations = [] |
7697
ef50074a0314
[repo api] deprecates source.extid2eid, only the system source should implement it,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7650
diff
changeset
|
428 |
kwargs = {'x': self.repo.eid2extid(self, entity.eid, session)} |
6652
592c88c8f018
[3.10] use cw_attr_cache, entity has no more iteritems method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6632
diff
changeset
|
429 |
for key, val in entity.cw_attr_cache.iteritems(): |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
430 |
relations.append('X %s %%(%s)s' % (key, key)) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
431 |
kwargs[key] = val |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
432 |
return relations, kwargs |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
433 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
434 |
def add_entity(self, session, entity): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
435 |
"""add a new entity to the source""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
436 |
raise NotImplementedError() |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
437 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
438 |
def update_entity(self, session, entity): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
439 |
"""update an entity in the source""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
440 |
relations, kwargs = self._entity_relations_and_kwargs(session, entity) |
7398
26695dd703d8
[repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7293
diff
changeset
|
441 |
cu = session.cnxset[self.uri] |
5363
d336476dcf57
[3.8 api update] remaining cache keys
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5323
diff
changeset
|
442 |
cu.execute('SET %s WHERE X eid %%(x)s' % ','.join(relations), kwargs) |
1805 | 443 |
self._query_cache.clear() |
7293
97505b798975
[entity 3.13 api] rename clear_all_caches into cw_clear_all_cache
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6958
diff
changeset
|
444 |
entity.cw_clear_all_caches() |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
445 |
|
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4816
diff
changeset
|
446 |
def delete_entity(self, session, entity): |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
447 |
"""delete an entity from the source""" |
7895 | 448 |
if session.deleted_in_transaction(self.eid): |
7650
278fe9c1f3ad
[repo] closes #1821172: fix source deletion
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7536
diff
changeset
|
449 |
# source is being deleted, don't propagate |
278fe9c1f3ad
[repo] closes #1821172: fix source deletion
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7536
diff
changeset
|
450 |
self._query_cache.clear() |
278fe9c1f3ad
[repo] closes #1821172: fix source deletion
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7536
diff
changeset
|
451 |
return |
7398
26695dd703d8
[repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7293
diff
changeset
|
452 |
cu = session.cnxset[self.uri] |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4816
diff
changeset
|
453 |
cu.execute('DELETE %s X WHERE X eid %%(x)s' % entity.__regid__, |
7697
ef50074a0314
[repo api] deprecates source.extid2eid, only the system source should implement it,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7650
diff
changeset
|
454 |
{'x': self.repo.eid2extid(self, entity.eid, session)}) |
1805 | 455 |
self._query_cache.clear() |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
456 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
457 |
def add_relation(self, session, subject, rtype, object): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
458 |
"""add a relation to the source""" |
7398
26695dd703d8
[repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7293
diff
changeset
|
459 |
cu = session.cnxset[self.uri] |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
460 |
cu.execute('SET X %s Y WHERE X eid %%(x)s, Y eid %%(y)s' % rtype, |
7697
ef50074a0314
[repo api] deprecates source.extid2eid, only the system source should implement it,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7650
diff
changeset
|
461 |
{'x': self.repo.eid2extid(self, subject, session), |
ef50074a0314
[repo api] deprecates source.extid2eid, only the system source should implement it,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7650
diff
changeset
|
462 |
'y': self.repo.eid2extid(self, object, session)}) |
1805 | 463 |
self._query_cache.clear() |
7293
97505b798975
[entity 3.13 api] rename clear_all_caches into cw_clear_all_cache
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6958
diff
changeset
|
464 |
session.entity_from_eid(subject).cw_clear_all_caches() |
97505b798975
[entity 3.13 api] rename clear_all_caches into cw_clear_all_cache
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6958
diff
changeset
|
465 |
session.entity_from_eid(object).cw_clear_all_caches() |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
466 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
467 |
def delete_relation(self, session, subject, rtype, object): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
468 |
"""delete a relation from the source""" |
7895 | 469 |
if session.deleted_in_transaction(self.eid): |
7650
278fe9c1f3ad
[repo] closes #1821172: fix source deletion
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7536
diff
changeset
|
470 |
# source is being deleted, don't propagate |
278fe9c1f3ad
[repo] closes #1821172: fix source deletion
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7536
diff
changeset
|
471 |
self._query_cache.clear() |
278fe9c1f3ad
[repo] closes #1821172: fix source deletion
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7536
diff
changeset
|
472 |
return |
7398
26695dd703d8
[repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7293
diff
changeset
|
473 |
cu = session.cnxset[self.uri] |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
474 |
cu.execute('DELETE X %s Y WHERE X eid %%(x)s, Y eid %%(y)s' % rtype, |
7697
ef50074a0314
[repo api] deprecates source.extid2eid, only the system source should implement it,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7650
diff
changeset
|
475 |
{'x': self.repo.eid2extid(self, subject, session), |
ef50074a0314
[repo api] deprecates source.extid2eid, only the system source should implement it,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7650
diff
changeset
|
476 |
'y': self.repo.eid2extid(self, object, session)}) |
1805 | 477 |
self._query_cache.clear() |
7293
97505b798975
[entity 3.13 api] rename clear_all_caches into cw_clear_all_cache
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6958
diff
changeset
|
478 |
session.entity_from_eid(subject).cw_clear_all_caches() |
97505b798975
[entity 3.13 api] rename clear_all_caches into cw_clear_all_cache
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6958
diff
changeset
|
479 |
session.entity_from_eid(object).cw_clear_all_caches() |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
480 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
481 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
482 |
class RQL2RQL(object): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
483 |
"""translate a local rql query to be executed on a distant repository""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
484 |
def __init__(self, source): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
485 |
self.source = source |
7697
ef50074a0314
[repo api] deprecates source.extid2eid, only the system source should implement it,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7650
diff
changeset
|
486 |
self.repo = source.repo |
390
739d12586b9d
avoid potentialy mis-interpreted attribute error
sylvain.thenault@logilab.fr
parents:
257
diff
changeset
|
487 |
self.current_operator = None |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
488 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
489 |
def _accept_children(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
490 |
res = [] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
491 |
for child in node.children: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
492 |
rql = child.accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
493 |
if rql is not None: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
494 |
res.append(rql) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
495 |
return res |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
496 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
497 |
def generate(self, session, rqlst, args): |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
498 |
self._session = session |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
499 |
self.kwargs = args |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
500 |
self.need_translation = False |
6655
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
501 |
self.has_local_eid = False |
5174
78438ad513ca
#759035: Automate addition of eid cachekey in RQL analysis
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
502 |
return self.visit_union(rqlst) |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
503 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
504 |
def visit_union(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
505 |
s = self._accept_children(node) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
506 |
if len(s) > 1: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
507 |
return ' UNION '.join('(%s)' % q for q in s) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
508 |
return s[0] |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
509 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
510 |
def visit_select(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
511 |
"""return the tree as an encoded rql string""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
512 |
self._varmaker = rqlvar_maker(defined=node.defined_vars.copy()) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
513 |
self._const_var = {} |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
514 |
if node.distinct: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
515 |
base = 'DISTINCT Any' |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
516 |
else: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
517 |
base = 'Any' |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
518 |
s = ['%s %s' % (base, ','.join(v.accept(self) for v in node.selection))] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
519 |
if node.groupby: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
520 |
s.append('GROUPBY %s' % ', '.join(group.accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
521 |
for group in node.groupby)) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
522 |
if node.orderby: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
523 |
s.append('ORDERBY %s' % ', '.join(self.visit_sortterm(term) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
524 |
for term in node.orderby)) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
525 |
if node.limit is not None: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
526 |
s.append('LIMIT %s' % node.limit) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
527 |
if node.offset: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
528 |
s.append('OFFSET %s' % node.offset) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
529 |
restrictions = [] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
530 |
if node.where is not None: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
531 |
nr = node.where.accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
532 |
if nr is not None: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
533 |
restrictions.append(nr) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
534 |
if restrictions: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
535 |
s.append('WHERE %s' % ','.join(restrictions)) |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
536 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
537 |
if node.having: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
538 |
s.append('HAVING %s' % ', '.join(term.accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
539 |
for term in node.having)) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
540 |
subqueries = [] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
541 |
for subquery in node.with_: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
542 |
subqueries.append('%s BEING (%s)' % (','.join(ca.name for ca in subquery.aliases), |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
543 |
self.visit_union(subquery.query))) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
544 |
if subqueries: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
545 |
s.append('WITH %s' % (','.join(subqueries))) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
546 |
return ' '.join(s) |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
547 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
548 |
def visit_and(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
549 |
res = self._accept_children(node) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
550 |
if res: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
551 |
return ', '.join(res) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
552 |
return |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
553 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
554 |
def visit_or(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
555 |
res = self._accept_children(node) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
556 |
if len(res) > 1: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
557 |
return ' OR '.join('(%s)' % rql for rql in res) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
558 |
elif res: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
559 |
return res[0] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
560 |
return |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
561 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
562 |
def visit_not(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
563 |
rql = node.children[0].accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
564 |
if rql: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
565 |
return 'NOT (%s)' % rql |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
566 |
return |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
567 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
568 |
def visit_exists(self, node): |
5582
3e133b29a1a4
[rql2sql] follow rql 0.26.1 changes: NOT nodes normalization, allowing simplification of sql generation, and fix #XXX
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5426
diff
changeset
|
569 |
rql = node.children[0].accept(self) |
3e133b29a1a4
[rql2sql] follow rql 0.26.1 changes: NOT nodes normalization, allowing simplification of sql generation, and fix #XXX
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5426
diff
changeset
|
570 |
if rql: |
3e133b29a1a4
[rql2sql] follow rql 0.26.1 changes: NOT nodes normalization, allowing simplification of sql generation, and fix #XXX
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5426
diff
changeset
|
571 |
return 'EXISTS(%s)' % rql |
3e133b29a1a4
[rql2sql] follow rql 0.26.1 changes: NOT nodes normalization, allowing simplification of sql generation, and fix #XXX
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5426
diff
changeset
|
572 |
return |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
573 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
574 |
def visit_relation(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
575 |
try: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
576 |
if isinstance(node.children[0], Constant): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
577 |
# simplified rqlst, reintroduce eid relation |
391
42693fe3ef6f
skip neged relation here as well
sylvain.thenault@logilab.fr
parents:
390
diff
changeset
|
578 |
try: |
42693fe3ef6f
skip neged relation here as well
sylvain.thenault@logilab.fr
parents:
390
diff
changeset
|
579 |
restr, lhs = self.process_eid_const(node.children[0]) |
42693fe3ef6f
skip neged relation here as well
sylvain.thenault@logilab.fr
parents:
390
diff
changeset
|
580 |
except UnknownEid: |
42693fe3ef6f
skip neged relation here as well
sylvain.thenault@logilab.fr
parents:
390
diff
changeset
|
581 |
# can safely skip not relation with an unsupported eid |
5582
3e133b29a1a4
[rql2sql] follow rql 0.26.1 changes: NOT nodes normalization, allowing simplification of sql generation, and fix #XXX
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5426
diff
changeset
|
582 |
if neged_relation(node): |
391
42693fe3ef6f
skip neged relation here as well
sylvain.thenault@logilab.fr
parents:
390
diff
changeset
|
583 |
return |
42693fe3ef6f
skip neged relation here as well
sylvain.thenault@logilab.fr
parents:
390
diff
changeset
|
584 |
raise |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
585 |
else: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
586 |
lhs = node.children[0].accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
587 |
restr = None |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
588 |
except UnknownEid: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
589 |
# can safely skip not relation with an unsupported eid |
5582
3e133b29a1a4
[rql2sql] follow rql 0.26.1 changes: NOT nodes normalization, allowing simplification of sql generation, and fix #XXX
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5426
diff
changeset
|
590 |
if neged_relation(node): |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
591 |
return |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
592 |
# XXX what about optional relation or outer NOT EXISTS() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
593 |
raise |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
594 |
if node.optional in ('left', 'both'): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
595 |
lhs += '?' |
3689
deb13e88e037
follow yams 0.25 api changes to improve performance
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3605
diff
changeset
|
596 |
if node.r_type == 'eid' or not self.source.schema.rschema(node.r_type).final: |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
597 |
self.need_translation = True |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
598 |
self.current_operator = node.operator() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
599 |
if isinstance(node.children[0], Constant): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
600 |
self.current_etypes = (node.children[0].uidtype,) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
601 |
else: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
602 |
self.current_etypes = node.children[0].variable.stinfo['possibletypes'] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
603 |
try: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
604 |
rhs = node.children[1].accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
605 |
except UnknownEid: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
606 |
# can safely skip not relation with an unsupported eid |
5582
3e133b29a1a4
[rql2sql] follow rql 0.26.1 changes: NOT nodes normalization, allowing simplification of sql generation, and fix #XXX
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5426
diff
changeset
|
607 |
if neged_relation(node): |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
608 |
return |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
609 |
# XXX what about optional relation or outer NOT EXISTS() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
610 |
raise |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
611 |
except ReplaceByInOperator, ex: |
1952
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1805
diff
changeset
|
612 |
rhs = 'IN (%s)' % ','.join(eid for eid in ex.eids) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
613 |
self.need_translation = False |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
614 |
self.current_operator = None |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
615 |
if node.optional in ('right', 'both'): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
616 |
rhs += '?' |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
617 |
if restr is not None: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
618 |
return '%s %s %s, %s' % (lhs, node.r_type, rhs, restr) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
619 |
return '%s %s %s' % (lhs, node.r_type, rhs) |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
620 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
621 |
def visit_comparison(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
622 |
if node.operator in ('=', 'IS'): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
623 |
return node.children[0].accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
624 |
return '%s %s' % (node.operator.encode(), |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
625 |
node.children[0].accept(self)) |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
626 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
627 |
def visit_mathexpression(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
628 |
return '(%s %s %s)' % (node.children[0].accept(self), |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
629 |
node.operator.encode(), |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
630 |
node.children[1].accept(self)) |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
631 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
632 |
def visit_function(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
633 |
#if node.name == 'IN': |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
634 |
res = [] |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
635 |
for child in node.children: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
636 |
try: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
637 |
rql = child.accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
638 |
except UnknownEid, ex: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
639 |
continue |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
640 |
res.append(rql) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
641 |
if not res: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
642 |
raise ex |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
643 |
return '%s(%s)' % (node.name, ', '.join(res)) |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
644 |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
645 |
def visit_constant(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
646 |
if self.need_translation or node.uidtype: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
647 |
if node.type == 'Int': |
6655
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
648 |
self.has_local_eid = True |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
649 |
return str(self.eid2extid(node.value)) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
650 |
if node.type == 'Substitute': |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
651 |
key = node.value |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
652 |
# ensure we have not yet translated the value... |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
653 |
if not key in self._const_var: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
654 |
self.kwargs[key] = self.eid2extid(self.kwargs[key]) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
655 |
self._const_var[key] = None |
6655
75112ff0511d
[ms pyro] skipping external entities is not enough else we may miss some relations with our configuration (eg X refinement_of Y WHERE X comes from cwo while Y comes from elo)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6652
diff
changeset
|
656 |
self.has_local_eid = True |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
657 |
return node.as_string() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
658 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
659 |
def visit_variableref(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
660 |
"""get the sql name for a variable reference""" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
661 |
return node.name |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
662 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
663 |
def visit_sortterm(self, node): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
664 |
if node.asc: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
665 |
return node.term.accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
666 |
return '%s DESC' % node.term.accept(self) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
667 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
668 |
def process_eid_const(self, const): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
669 |
value = const.eval(self.kwargs) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
670 |
try: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
671 |
return None, self._const_var[value] |
7815
2a164a9cf81c
[exceptions] stop catching any exception in various places (closes #1942716)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7697
diff
changeset
|
672 |
except Exception: |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
673 |
var = self._varmaker.next() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
674 |
self.need_translation = True |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
675 |
restr = '%s eid %s' % (var, self.visit_constant(const)) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
676 |
self.need_translation = False |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
677 |
self._const_var[value] = var |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
678 |
return restr, var |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
679 |
|
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
680 |
def eid2extid(self, eid): |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
681 |
try: |
7697
ef50074a0314
[repo api] deprecates source.extid2eid, only the system source should implement it,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7650
diff
changeset
|
682 |
return self.repo.eid2extid(self.source, eid, self._session) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
683 |
except UnknownEid: |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
684 |
operator = self.current_operator |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
685 |
if operator is not None and operator != '=': |
1954 | 686 |
# deal with query like "X eid > 12" |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
687 |
# |
1954 | 688 |
# The problem is that eid order in the external source may |
689 |
# differ from the local source |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
690 |
# |
1954 | 691 |
# So search for all eids from this source matching the condition |
692 |
# locally and then to replace the "> 12" branch by "IN (eids)" |
|
693 |
# |
|
694 |
# XXX we may have to insert a huge number of eids...) |
|
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
695 |
sql = "SELECT extid FROM entities WHERE source='%s' AND type IN (%s) AND eid%s%s" |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
696 |
etypes = ','.join("'%s'" % etype for etype in self.current_etypes) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
697 |
cu = self._session.system_sql(sql % (self.source.uri, etypes, |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
698 |
operator, eid)) |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
699 |
# XXX buggy cu.rowcount which may be zero while there are some |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
700 |
# results |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
701 |
rows = cu.fetchall() |
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
702 |
if rows: |
1952
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1805
diff
changeset
|
703 |
raise ReplaceByInOperator((b64decode(r[0]) for r in rows)) |
257
4c7d3af7e94d
restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
704 |
raise |
1791
c77629112437
clear cache before synchronization
sylvain.thenault@logilab.fr
parents:
1398
diff
changeset
|
705 |