author | Adrien Di Mascio <Adrien.DiMascio@logilab.fr> |
Thu, 24 Sep 2009 00:24:21 +0200 | |
changeset 3460 | e4843535db25 |
parent 3399 | 2b84f4adb6f8 |
child 3503 | 06bced8edddf |
permissions | -rw-r--r-- |
0 | 1 |
"""Adapters for native cubicweb sources. |
2 |
||
1952
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
3 |
Notes: |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
4 |
* extid (aka external id, the primary key of an entity in the external source |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
5 |
from which it comes from) are stored in a varchar column encoded as a base64 |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
6 |
string. This is because it should actually be Bytes but we want an index on |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
7 |
it for fast querying. |
2056 | 8 |
|
0 | 9 |
:organization: Logilab |
1977
606923dff11b
big bunch of copyright / docstring update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1954
diff
changeset
|
10 |
:copyright: 2001-2009 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. |
0 | 11 |
:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr |
1977
606923dff11b
big bunch of copyright / docstring update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1954
diff
changeset
|
12 |
:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses |
0 | 13 |
""" |
14 |
__docformat__ = "restructuredtext en" |
|
15 |
||
16 |
from threading import Lock |
|
1016
26387b836099
use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents:
973
diff
changeset
|
17 |
from datetime import datetime |
1952
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
18 |
from base64 import b64decode, b64encode |
0 | 19 |
|
20 |
from logilab.common.cache import Cache |
|
21 |
from logilab.common.configuration import REQUIRED |
|
22 |
from logilab.common.adbh import get_adv_func_helper |
|
23 |
||
24 |
from indexer import get_indexer |
|
25 |
||
26 |
from cubicweb import UnknownEid, AuthenticationError, Binary, server |
|
27 |
from cubicweb.server.utils import crypt_password |
|
2759
23d7a75693f8
R refactor backup and use tar.gz to store all sources
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2638
diff
changeset
|
28 |
from cubicweb.server.sqlutils import SQL_PREFIX, SQLAdapterMixIn |
0 | 29 |
from cubicweb.server.rqlannotation import set_qdata |
2625
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2620
diff
changeset
|
30 |
from cubicweb.server.sources import AbstractSource, dbg_st_search, dbg_results |
0 | 31 |
from cubicweb.server.sources.rql2sql import SQLGenerator |
32 |
||
33 |
||
2354
9b4bac626977
ability to map attributes to something else than usual cw mapping on sql generation
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2310
diff
changeset
|
34 |
ATTR_MAP = {} |
0 | 35 |
NONSYSTEM_ETYPES = set() |
36 |
NONSYSTEM_RELATIONS = set() |
|
37 |
||
38 |
class LogCursor(object): |
|
39 |
def __init__(self, cursor): |
|
40 |
self.cu = cursor |
|
1792 | 41 |
|
0 | 42 |
def execute(self, query, args=None): |
43 |
"""Execute a query. |
|
44 |
it's a function just so that it shows up in profiling |
|
45 |
""" |
|
2593
16d9419a4a79
F: start to handle binary debug log level on the server side
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2589
diff
changeset
|
46 |
if server.DEBUG & server.DBG_SQL: |
0 | 47 |
print 'exec', query, args |
48 |
try: |
|
49 |
self.cu.execute(str(query), args) |
|
50 |
except Exception, ex: |
|
51 |
print "sql: %r\n args: %s\ndbms message: %r" % ( |
|
52 |
query, args, ex.args[0]) |
|
53 |
raise |
|
1792 | 54 |
|
0 | 55 |
def fetchall(self): |
56 |
return self.cu.fetchall() |
|
1792 | 57 |
|
0 | 58 |
def fetchone(self): |
59 |
return self.cu.fetchone() |
|
1792 | 60 |
|
2625
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2620
diff
changeset
|
61 |
|
0 | 62 |
def make_schema(selected, solution, table, typemap): |
63 |
"""return a sql schema to store RQL query result""" |
|
64 |
sql = [] |
|
65 |
varmap = {} |
|
66 |
for i, term in enumerate(selected): |
|
67 |
name = 'C%s' % i |
|
68 |
key = term.as_string() |
|
69 |
varmap[key] = '%s.%s' % (table, name) |
|
70 |
ttype = term.get_type(solution) |
|
71 |
try: |
|
72 |
sql.append('%s %s' % (name, typemap[ttype])) |
|
73 |
except KeyError: |
|
74 |
# assert not schema(ttype).is_final() |
|
75 |
sql.append('%s %s' % (name, typemap['Int'])) |
|
76 |
return ','.join(sql), varmap |
|
77 |
||
2625
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2620
diff
changeset
|
78 |
|
0 | 79 |
def _modified_sql(table, etypes): |
80 |
# XXX protect against sql injection |
|
81 |
if len(etypes) > 1: |
|
82 |
restr = 'type IN (%s)' % ','.join("'%s'" % etype for etype in etypes) |
|
83 |
else: |
|
84 |
restr = "type='%s'" % etypes[0] |
|
85 |
if table == 'entities': |
|
86 |
attr = 'mtime' |
|
87 |
else: |
|
88 |
attr = 'dtime' |
|
89 |
return 'SELECT type, eid FROM %s WHERE %s AND %s > %%(time)s' % ( |
|
90 |
table, restr, attr) |
|
91 |
||
92 |
||
93 |
class NativeSQLSource(SQLAdapterMixIn, AbstractSource): |
|
94 |
"""adapter for source using the native cubicweb schema (see below) |
|
95 |
""" |
|
2354
9b4bac626977
ability to map attributes to something else than usual cw mapping on sql generation
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2310
diff
changeset
|
96 |
sqlgen_class = SQLGenerator |
1792 | 97 |
|
1398
5fe84a5f7035
rename internal entity types to have CW prefix instead of E
sylvain.thenault@logilab.fr
parents:
1263
diff
changeset
|
98 |
passwd_rql = "Any P WHERE X is CWUser, X login %(login)s, X upassword P" |
5fe84a5f7035
rename internal entity types to have CW prefix instead of E
sylvain.thenault@logilab.fr
parents:
1263
diff
changeset
|
99 |
auth_rql = "Any X WHERE X is CWUser, X login %(login)s, X upassword %(pwd)s" |
5fe84a5f7035
rename internal entity types to have CW prefix instead of E
sylvain.thenault@logilab.fr
parents:
1263
diff
changeset
|
100 |
_sols = ({'X': 'CWUser', 'P': 'Password'},) |
1792 | 101 |
|
0 | 102 |
options = ( |
103 |
('db-driver', |
|
104 |
{'type' : 'string', |
|
105 |
'default': 'postgres', |
|
106 |
'help': 'database driver (postgres or sqlite)', |
|
107 |
'group': 'native-source', 'inputlevel': 1, |
|
108 |
}), |
|
109 |
('db-host', |
|
110 |
{'type' : 'string', |
|
111 |
'default': '', |
|
112 |
'help': 'database host', |
|
113 |
'group': 'native-source', 'inputlevel': 1, |
|
114 |
}), |
|
2566
714a8743d423
missing db-port option to source's option definitions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2354
diff
changeset
|
115 |
('db-port', |
714a8743d423
missing db-port option to source's option definitions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2354
diff
changeset
|
116 |
{'type' : 'string', |
714a8743d423
missing db-port option to source's option definitions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2354
diff
changeset
|
117 |
'default': '', |
714a8743d423
missing db-port option to source's option definitions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2354
diff
changeset
|
118 |
'help': 'database port', |
714a8743d423
missing db-port option to source's option definitions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2354
diff
changeset
|
119 |
'group': 'native-source', 'inputlevel': 1, |
714a8743d423
missing db-port option to source's option definitions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2354
diff
changeset
|
120 |
}), |
0 | 121 |
('db-name', |
122 |
{'type' : 'string', |
|
123 |
'default': REQUIRED, |
|
124 |
'help': 'database name', |
|
125 |
'group': 'native-source', 'inputlevel': 0, |
|
126 |
}), |
|
127 |
('db-user', |
|
128 |
{'type' : 'string', |
|
129 |
'default': 'cubicweb', |
|
130 |
'help': 'database user', |
|
131 |
'group': 'native-source', 'inputlevel': 0, |
|
132 |
}), |
|
133 |
('db-password', |
|
134 |
{'type' : 'password', |
|
135 |
'default': '', |
|
136 |
'help': 'database password', |
|
137 |
'group': 'native-source', 'inputlevel': 0, |
|
138 |
}), |
|
139 |
('db-encoding', |
|
140 |
{'type' : 'string', |
|
141 |
'default': 'utf8', |
|
142 |
'help': 'database encoding', |
|
143 |
'group': 'native-source', 'inputlevel': 1, |
|
144 |
}), |
|
145 |
) |
|
1792 | 146 |
|
0 | 147 |
def __init__(self, repo, appschema, source_config, *args, **kwargs): |
148 |
SQLAdapterMixIn.__init__(self, source_config) |
|
149 |
AbstractSource.__init__(self, repo, appschema, source_config, |
|
150 |
*args, **kwargs) |
|
151 |
# sql generator |
|
2354
9b4bac626977
ability to map attributes to something else than usual cw mapping on sql generation
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2310
diff
changeset
|
152 |
self._rql_sqlgen = self.sqlgen_class(appschema, self.dbhelper, |
9b4bac626977
ability to map attributes to something else than usual cw mapping on sql generation
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2310
diff
changeset
|
153 |
self.encoding, ATTR_MAP.copy()) |
0 | 154 |
# full text index helper |
155 |
self.indexer = get_indexer(self.dbdriver, self.encoding) |
|
156 |
# advanced functionality helper |
|
157 |
self.dbhelper.fti_uid_attr = self.indexer.uid_attr |
|
158 |
self.dbhelper.fti_table = self.indexer.table |
|
159 |
self.dbhelper.fti_restriction_sql = self.indexer.restriction_sql |
|
160 |
self.dbhelper.fti_need_distinct_query = self.indexer.need_distinct |
|
161 |
# sql queries cache |
|
162 |
self._cache = Cache(repo.config['rql-cache-size']) |
|
163 |
self._temp_table_data = {} |
|
164 |
self._eid_creation_lock = Lock() |
|
2072
8008e8812d76
deactivate sqlite connection wrapping for unittest_multisources for now
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2066
diff
changeset
|
165 |
# XXX no_sqlite_wrap trick since we've a sqlite locking pb when |
8008e8812d76
deactivate sqlite connection wrapping for unittest_multisources for now
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2066
diff
changeset
|
166 |
# running unittest_multisources with the wrapping below |
8008e8812d76
deactivate sqlite connection wrapping for unittest_multisources for now
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2066
diff
changeset
|
167 |
if self.dbdriver == 'sqlite' and \ |
8008e8812d76
deactivate sqlite connection wrapping for unittest_multisources for now
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2066
diff
changeset
|
168 |
not getattr(repo.config, 'no_sqlite_wrap', False): |
2064
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
169 |
from cubicweb.server.sources.extlite import ConnectionWrapper |
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
170 |
self.get_connection = lambda: ConnectionWrapper(self) |
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
171 |
self.check_connection = lambda cnx: cnx |
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
172 |
def pool_reset(cnx): |
2620
de68f84b8f54
R [sql source] cnx._cnx checked in cnx.close, don't do it here
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2619
diff
changeset
|
173 |
cnx.close() |
2064
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
174 |
self.pool_reset = pool_reset |
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
175 |
|
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
176 |
@property |
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
177 |
def _sqlcnx(self): |
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
178 |
# XXX: sqlite connections can only be used in the same thread, so |
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
179 |
# create a new one each time necessary. If it appears to be time |
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
180 |
# consuming, find another way |
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
181 |
return SQLAdapterMixIn.get_connection(self) |
0 | 182 |
|
183 |
def reset_caches(self): |
|
184 |
"""method called during test to reset potential source caches""" |
|
185 |
self._cache = Cache(self.repo.config['rql-cache-size']) |
|
1792 | 186 |
|
0 | 187 |
def clear_eid_cache(self, eid, etype): |
188 |
"""clear potential caches for the given eid""" |
|
2610
2933cc6bf9ad
[F native source] fix attempts to clear cache key
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2593
diff
changeset
|
189 |
self._cache.pop('Any X WHERE X eid %s, X is %s' % (eid, etype), None) |
0 | 190 |
self._cache.pop('Any X WHERE X eid %s' % eid, None) |
2610
2933cc6bf9ad
[F native source] fix attempts to clear cache key
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2593
diff
changeset
|
191 |
self._cache.pop('Any %s' % eid, None) |
1792 | 192 |
|
0 | 193 |
def sqlexec(self, session, sql, args=None): |
194 |
"""execute the query and return its result""" |
|
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
195 |
return self.process_result(self.doexec(session, sql, args)) |
1792 | 196 |
|
0 | 197 |
def init_creating(self): |
2064
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
198 |
pool = self.repo._get_pool() |
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
199 |
pool.pool_set() |
0 | 200 |
# check full text index availibility |
201 |
if not self.indexer.has_fti_table(pool['system']): |
|
202 |
self.error('no text index table') |
|
203 |
self.indexer = None |
|
2064
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
204 |
pool.pool_reset() |
0 | 205 |
self.repo._free_pool(pool) |
206 |
||
2759
23d7a75693f8
R refactor backup and use tar.gz to store all sources
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2638
diff
changeset
|
207 |
def backup(self, backupfile): |
23d7a75693f8
R refactor backup and use tar.gz to store all sources
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2638
diff
changeset
|
208 |
"""method called to create a backup of the source's data""" |
23d7a75693f8
R refactor backup and use tar.gz to store all sources
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2638
diff
changeset
|
209 |
self.close_pool_connections() |
23d7a75693f8
R refactor backup and use tar.gz to store all sources
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2638
diff
changeset
|
210 |
try: |
23d7a75693f8
R refactor backup and use tar.gz to store all sources
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2638
diff
changeset
|
211 |
self.backup_to_file(backupfile) |
23d7a75693f8
R refactor backup and use tar.gz to store all sources
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2638
diff
changeset
|
212 |
finally: |
23d7a75693f8
R refactor backup and use tar.gz to store all sources
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2638
diff
changeset
|
213 |
self.open_pool_connections() |
2493
9806571ea790
major refactoring of database dump/restore:
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2476
diff
changeset
|
214 |
|
2759
23d7a75693f8
R refactor backup and use tar.gz to store all sources
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2638
diff
changeset
|
215 |
def restore(self, backupfile, drop): |
2493
9806571ea790
major refactoring of database dump/restore:
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2476
diff
changeset
|
216 |
"""method called to restore a backup of source's data""" |
2959
daabb9bc5233
make db-restore command work even with no/corrupted database
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2759
diff
changeset
|
217 |
if self.repo.config.open_connections_pools: |
daabb9bc5233
make db-restore command work even with no/corrupted database
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2759
diff
changeset
|
218 |
self.close_pool_connections() |
2759
23d7a75693f8
R refactor backup and use tar.gz to store all sources
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2638
diff
changeset
|
219 |
try: |
23d7a75693f8
R refactor backup and use tar.gz to store all sources
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2638
diff
changeset
|
220 |
self.restore_from_file(backupfile, drop) |
23d7a75693f8
R refactor backup and use tar.gz to store all sources
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2638
diff
changeset
|
221 |
finally: |
2959
daabb9bc5233
make db-restore command work even with no/corrupted database
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2759
diff
changeset
|
222 |
if self.repo.config.open_connections_pools: |
daabb9bc5233
make db-restore command work even with no/corrupted database
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2759
diff
changeset
|
223 |
self.open_pool_connections() |
2493
9806571ea790
major refactoring of database dump/restore:
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2476
diff
changeset
|
224 |
|
0 | 225 |
def init(self): |
1792 | 226 |
self.init_creating() |
227 |
||
2354
9b4bac626977
ability to map attributes to something else than usual cw mapping on sql generation
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2310
diff
changeset
|
228 |
def map_attribute(self, etype, attr, cb): |
9b4bac626977
ability to map attributes to something else than usual cw mapping on sql generation
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2310
diff
changeset
|
229 |
self._rql_sqlgen.attr_map['%s.%s' % (etype, attr)] = cb |
2493
9806571ea790
major refactoring of database dump/restore:
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2476
diff
changeset
|
230 |
|
0 | 231 |
# ISource interface ####################################################### |
232 |
||
233 |
def compile_rql(self, rql): |
|
3240
8604a15995d1
refactor so that rql rewriter may be used outside the server. Enhance it to be usable for RRQLExpression as well
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3041
diff
changeset
|
234 |
rqlst = self.repo.vreg.rqlhelper.parse(rql) |
0 | 235 |
rqlst.restricted_vars = () |
236 |
rqlst.children[0].solutions = self._sols |
|
237 |
self.repo.querier.sqlgen_annotate(rqlst) |
|
438 | 238 |
set_qdata(self.schema.rschema, rqlst, ()) |
0 | 239 |
return rqlst |
1792 | 240 |
|
0 | 241 |
def set_schema(self, schema): |
2476
1294a6bdf3bf
application -> instance where it makes sense
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2354
diff
changeset
|
242 |
"""set the instance'schema""" |
0 | 243 |
self._cache = Cache(self.repo.config['rql-cache-size']) |
244 |
self.cache_hit, self.cache_miss, self.no_cache = 0, 0, 0 |
|
245 |
self.schema = schema |
|
246 |
try: |
|
247 |
self._rql_sqlgen.schema = schema |
|
248 |
except AttributeError: |
|
249 |
pass # __init__ |
|
1398
5fe84a5f7035
rename internal entity types to have CW prefix instead of E
sylvain.thenault@logilab.fr
parents:
1263
diff
changeset
|
250 |
if 'CWUser' in schema: # probably an empty schema if not true... |
0 | 251 |
# rql syntax trees used to authenticate users |
252 |
self._passwd_rqlst = self.compile_rql(self.passwd_rql) |
|
253 |
self._auth_rqlst = self.compile_rql(self.auth_rql) |
|
1792 | 254 |
|
0 | 255 |
def support_entity(self, etype, write=False): |
256 |
"""return true if the given entity's type is handled by this adapter |
|
257 |
if write is true, return true only if it's a RW support |
|
258 |
""" |
|
259 |
return not etype in NONSYSTEM_ETYPES |
|
1792 | 260 |
|
0 | 261 |
def support_relation(self, rtype, write=False): |
262 |
"""return true if the given relation's type is handled by this adapter |
|
263 |
if write is true, return true only if it's a RW support |
|
264 |
""" |
|
265 |
if write: |
|
266 |
return not rtype in NONSYSTEM_RELATIONS |
|
267 |
# due to current multi-sources implementation, the system source |
|
1792 | 268 |
# can't claim not supporting a relation |
0 | 269 |
return True #not rtype == 'content_for' |
270 |
||
3041
782fa7566a22
[multi-sources] new may_cross_relation method on sources
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2967
diff
changeset
|
271 |
def may_cross_relation(self, rtype): |
782fa7566a22
[multi-sources] new may_cross_relation method on sources
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2967
diff
changeset
|
272 |
return True |
782fa7566a22
[multi-sources] new may_cross_relation method on sources
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2967
diff
changeset
|
273 |
|
0 | 274 |
def authenticate(self, session, login, password): |
1398
5fe84a5f7035
rename internal entity types to have CW prefix instead of E
sylvain.thenault@logilab.fr
parents:
1263
diff
changeset
|
275 |
"""return CWUser eid for the given login/password if this account is |
0 | 276 |
defined in this source, else raise `AuthenticationError` |
277 |
||
278 |
two queries are needed since passwords are stored crypted, so we have |
|
279 |
to fetch the salt first |
|
280 |
""" |
|
281 |
args = {'login': login, 'pwd' : password} |
|
282 |
if password is not None: |
|
283 |
rset = self.syntax_tree_search(session, self._passwd_rqlst, args) |
|
284 |
try: |
|
285 |
pwd = rset[0][0] |
|
286 |
except IndexError: |
|
287 |
raise AuthenticationError('bad login') |
|
1954 | 288 |
# passwords are stored using the Bytes type, so we get a StringIO |
0 | 289 |
if pwd is not None: |
290 |
args['pwd'] = crypt_password(password, pwd.getvalue()[:2]) |
|
291 |
# get eid from login and (crypted) password |
|
292 |
rset = self.syntax_tree_search(session, self._auth_rqlst, args) |
|
293 |
try: |
|
294 |
return rset[0][0] |
|
295 |
except IndexError: |
|
296 |
raise AuthenticationError('bad password') |
|
1792 | 297 |
|
298 |
def syntax_tree_search(self, session, union, args=None, cachekey=None, |
|
0 | 299 |
varmap=None): |
300 |
"""return result from this source for a rql query (actually from |
|
301 |
a rql syntax tree and a solution dictionary mapping each used |
|
302 |
variable to a possible type). If cachekey is given, the query |
|
303 |
necessary to fetch the results (but not the results themselves) |
|
304 |
may be cached using this key. |
|
305 |
""" |
|
2625
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2620
diff
changeset
|
306 |
assert dbg_st_search(self.uri, union, varmap, args, cachekey) |
0 | 307 |
# remember number of actually selected term (sql generation may append some) |
308 |
if cachekey is None: |
|
309 |
self.no_cache += 1 |
|
310 |
# generate sql query if we are able to do so (not supported types...) |
|
311 |
sql, query_args = self._rql_sqlgen.generate(union, args, varmap) |
|
312 |
else: |
|
313 |
# sql may be cached |
|
314 |
try: |
|
315 |
sql, query_args = self._cache[cachekey] |
|
316 |
self.cache_hit += 1 |
|
317 |
except KeyError: |
|
318 |
self.cache_miss += 1 |
|
319 |
sql, query_args = self._rql_sqlgen.generate(union, args, varmap) |
|
320 |
self._cache[cachekey] = sql, query_args |
|
321 |
args = self.merge_args(args, query_args) |
|
322 |
assert isinstance(sql, basestring), repr(sql) |
|
323 |
try: |
|
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
324 |
cursor = self.doexec(session, sql, args) |
0 | 325 |
except (self.dbapi_module.OperationalError, |
326 |
self.dbapi_module.InterfaceError): |
|
327 |
# FIXME: better detection of deconnection pb |
|
328 |
self.info("request failed '%s' ... retry with a new cursor", sql) |
|
329 |
session.pool.reconnect(self) |
|
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
330 |
cursor = self.doexec(session, sql, args) |
2625
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2620
diff
changeset
|
331 |
results = self.process_result(cursor) |
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2620
diff
changeset
|
332 |
assert dbg_results(results) |
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2620
diff
changeset
|
333 |
return results |
1792 | 334 |
|
0 | 335 |
def flying_insert(self, table, session, union, args=None, varmap=None): |
336 |
"""similar as .syntax_tree_search, but inserts data in the |
|
337 |
temporary table (on-the-fly if possible, eg for the system |
|
338 |
source whose the given cursor come from). If not possible, |
|
339 |
inserts all data by calling .executemany(). |
|
340 |
""" |
|
2625
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2620
diff
changeset
|
341 |
assert dbg_st_search( |
2638 | 342 |
self.uri, union, varmap, args, |
2625
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2620
diff
changeset
|
343 |
prefix='ON THE FLY temp data insertion into %s from' % table) |
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2620
diff
changeset
|
344 |
# generate sql queries if we are able to do so |
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2620
diff
changeset
|
345 |
sql, query_args = self._rql_sqlgen.generate(union, args, varmap) |
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2620
diff
changeset
|
346 |
query = 'INSERT INTO %s %s' % (table, sql.encode(self.encoding)) |
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2620
diff
changeset
|
347 |
self.doexec(session, query, self.merge_args(args, query_args)) |
1792 | 348 |
|
2627
d710278e0c1c
manual_insert is a public method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2625
diff
changeset
|
349 |
def manual_insert(self, results, table, session): |
0 | 350 |
"""insert given result into a temporary table on the system source""" |
2625
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2620
diff
changeset
|
351 |
if server.DEBUG & server.DBG_RQL: |
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2620
diff
changeset
|
352 |
print ' manual insertion of', res, 'into', table |
0 | 353 |
if not results: |
354 |
return |
|
355 |
query_args = ['%%(%s)s' % i for i in xrange(len(results[0]))] |
|
356 |
query = 'INSERT INTO %s VALUES(%s)' % (table, ','.join(query_args)) |
|
357 |
kwargs_list = [] |
|
358 |
for row in results: |
|
359 |
kwargs = {} |
|
360 |
row = tuple(row) |
|
361 |
for index, cell in enumerate(row): |
|
2066
2c4bf4ee88a2
cleanup, stop encoding unicode string in manual_insert, no more necessary and make crash recent sqlite w/ 8bit string
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2064
diff
changeset
|
362 |
if isinstance(cell, Binary): |
0 | 363 |
cell = self.binary(cell.getvalue()) |
364 |
kwargs[str(index)] = cell |
|
365 |
kwargs_list.append(kwargs) |
|
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
366 |
self.doexecmany(session, query, kwargs_list) |
0 | 367 |
|
368 |
def clean_temp_data(self, session, temptables): |
|
369 |
"""remove temporary data, usually associated to temporary tables""" |
|
370 |
if temptables: |
|
371 |
for table in temptables: |
|
372 |
try: |
|
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
373 |
self.doexec(session,'DROP TABLE %s' % table) |
0 | 374 |
except: |
375 |
pass |
|
376 |
try: |
|
377 |
del self._temp_table_data[table] |
|
378 |
except KeyError: |
|
379 |
continue |
|
1792 | 380 |
|
0 | 381 |
def add_entity(self, session, entity): |
382 |
"""add a new entity to the source""" |
|
383 |
attrs = self.preprocess_entity(entity) |
|
1251
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1079
diff
changeset
|
384 |
sql = self.sqlgen.insert(SQL_PREFIX + str(entity.e_schema), attrs) |
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
385 |
self.doexec(session, sql, attrs) |
1792 | 386 |
|
0 | 387 |
def update_entity(self, session, entity): |
388 |
"""replace an entity in the source""" |
|
389 |
attrs = self.preprocess_entity(entity) |
|
1251
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1079
diff
changeset
|
390 |
sql = self.sqlgen.update(SQL_PREFIX + str(entity.e_schema), attrs, [SQL_PREFIX + 'eid']) |
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
391 |
self.doexec(session, sql, attrs) |
0 | 392 |
|
393 |
def delete_entity(self, session, etype, eid): |
|
394 |
"""delete an entity from the source""" |
|
1251
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1079
diff
changeset
|
395 |
attrs = {SQL_PREFIX + 'eid': eid} |
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1079
diff
changeset
|
396 |
sql = self.sqlgen.delete(SQL_PREFIX + etype, attrs) |
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
397 |
self.doexec(session, sql, attrs) |
0 | 398 |
|
399 |
def add_relation(self, session, subject, rtype, object): |
|
400 |
"""add a relation to the source""" |
|
401 |
attrs = {'eid_from': subject, 'eid_to': object} |
|
402 |
sql = self.sqlgen.insert('%s_relation' % rtype, attrs) |
|
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
403 |
self.doexec(session, sql, attrs) |
1792 | 404 |
|
0 | 405 |
def delete_relation(self, session, subject, rtype, object): |
406 |
"""delete a relation from the source""" |
|
407 |
rschema = self.schema.rschema(rtype) |
|
408 |
if rschema.inlined: |
|
1251
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1079
diff
changeset
|
409 |
table = SQL_PREFIX + session.describe(subject)[0] |
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1079
diff
changeset
|
410 |
column = SQL_PREFIX + rtype |
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1079
diff
changeset
|
411 |
sql = 'UPDATE %s SET %s=NULL WHERE %seid=%%(eid)s' % (table, column, |
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1079
diff
changeset
|
412 |
SQL_PREFIX) |
0 | 413 |
attrs = {'eid' : subject} |
414 |
else: |
|
415 |
attrs = {'eid_from': subject, 'eid_to': object} |
|
416 |
sql = self.sqlgen.delete('%s_relation' % rtype, attrs) |
|
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
417 |
self.doexec(session, sql, attrs) |
0 | 418 |
|
2618
ff9b0d5bd884
[F repo sqlite schema changes] don't rollback on potentially expected schema changes failure
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2611
diff
changeset
|
419 |
def doexec(self, session, query, args=None, rollback=True): |
0 | 420 |
"""Execute a query. |
421 |
it's a function just so that it shows up in profiling |
|
422 |
""" |
|
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
423 |
cursor = session.pool[self.uri] |
2593
16d9419a4a79
F: start to handle binary debug log level on the server side
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2589
diff
changeset
|
424 |
if server.DEBUG & server.DBG_SQL: |
2625
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2620
diff
changeset
|
425 |
cnx = session.pool.connection(self.uri) |
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2620
diff
changeset
|
426 |
# getattr to get the actual connection if cnx is a ConnectionWrapper |
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2620
diff
changeset
|
427 |
# instance |
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2620
diff
changeset
|
428 |
print 'exec', query, args, getattr(cnx, '_cnx', cnx) |
0 | 429 |
try: |
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
430 |
# str(query) to avoid error if it's an unicode string |
0 | 431 |
cursor.execute(str(query), args) |
432 |
except Exception, ex: |
|
433 |
self.critical("sql: %r\n args: %s\ndbms message: %r", |
|
434 |
query, args, ex.args[0]) |
|
2618
ff9b0d5bd884
[F repo sqlite schema changes] don't rollback on potentially expected schema changes failure
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2611
diff
changeset
|
435 |
if rollback: |
ff9b0d5bd884
[F repo sqlite schema changes] don't rollback on potentially expected schema changes failure
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2611
diff
changeset
|
436 |
try: |
ff9b0d5bd884
[F repo sqlite schema changes] don't rollback on potentially expected schema changes failure
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2611
diff
changeset
|
437 |
session.pool.connection(self.uri).rollback() |
ff9b0d5bd884
[F repo sqlite schema changes] don't rollback on potentially expected schema changes failure
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2611
diff
changeset
|
438 |
self.critical('transaction has been rollbacked') |
ff9b0d5bd884
[F repo sqlite schema changes] don't rollback on potentially expected schema changes failure
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2611
diff
changeset
|
439 |
except: |
ff9b0d5bd884
[F repo sqlite schema changes] don't rollback on potentially expected schema changes failure
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2611
diff
changeset
|
440 |
pass |
0 | 441 |
raise |
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
442 |
return cursor |
1792 | 443 |
|
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
444 |
def doexecmany(self, session, query, args): |
0 | 445 |
"""Execute a query. |
446 |
it's a function just so that it shows up in profiling |
|
447 |
""" |
|
2593
16d9419a4a79
F: start to handle binary debug log level on the server side
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2589
diff
changeset
|
448 |
if server.DEBUG & server.DBG_SQL: |
0 | 449 |
print 'execmany', query, 'with', len(args), 'arguments' |
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
450 |
cursor = session.pool[self.uri] |
0 | 451 |
try: |
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
452 |
# str(query) to avoid error if it's an unicode string |
0 | 453 |
cursor.executemany(str(query), args) |
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
454 |
except Exception, ex: |
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
455 |
self.critical("sql many: %r\n args: %s\ndbms message: %r", |
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
456 |
query, args, ex.args[0]) |
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
457 |
try: |
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
458 |
session.pool.connection(self.uri).rollback() |
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
459 |
self.critical('transaction has been rollbacked') |
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
460 |
except: |
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
461 |
pass |
0 | 462 |
raise |
1792 | 463 |
|
0 | 464 |
# short cut to method requiring advanced db helper usage ################## |
1792 | 465 |
|
0 | 466 |
def create_index(self, session, table, column, unique=False): |
467 |
cursor = LogCursor(session.pool[self.uri]) |
|
468 |
self.dbhelper.create_index(cursor, table, column, unique) |
|
1792 | 469 |
|
0 | 470 |
def drop_index(self, session, table, column, unique=False): |
471 |
cursor = LogCursor(session.pool[self.uri]) |
|
472 |
self.dbhelper.drop_index(cursor, table, column, unique) |
|
473 |
||
474 |
# system source interface ################################################# |
|
475 |
||
476 |
def eid_type_source(self, session, eid): |
|
477 |
"""return a tuple (type, source, extid) for the entity with id <eid>""" |
|
478 |
sql = 'SELECT type, source, extid FROM entities WHERE eid=%s' % eid |
|
479 |
try: |
|
480 |
res = session.system_sql(sql).fetchone() |
|
481 |
except: |
|
1079
452cb76fe07a
backport typo fix
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
973
diff
changeset
|
482 |
assert session.pool, 'session has no pool set' |
0 | 483 |
raise UnknownEid(eid) |
484 |
if res is None: |
|
485 |
raise UnknownEid(eid) |
|
1952
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
486 |
if res[-1] is not None: |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
487 |
if not isinstance(res, list): |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
488 |
res = list(res) |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
489 |
res[-1] = b64decode(res[-1]) |
0 | 490 |
return res |
491 |
||
1952
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
492 |
def extid2eid(self, session, source, extid): |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
493 |
"""get eid from an external id. Return None if no record found.""" |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
494 |
assert isinstance(extid, str) |
0 | 495 |
cursor = session.system_sql('SELECT eid FROM entities WHERE ' |
496 |
'extid=%(x)s AND source=%(s)s', |
|
1952
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
497 |
{'x': b64encode(extid), 's': source.uri}) |
0 | 498 |
# XXX testing rowcount cause strange bug with sqlite, results are there |
499 |
# but rowcount is 0 |
|
1792 | 500 |
#if cursor.rowcount > 0: |
0 | 501 |
try: |
502 |
result = cursor.fetchone() |
|
503 |
if result: |
|
1954 | 504 |
return result[0] |
0 | 505 |
except: |
506 |
pass |
|
507 |
return None |
|
1792 | 508 |
|
0 | 509 |
def temp_table_def(self, selected, sol, table): |
510 |
return make_schema(selected, sol, table, self.dbhelper.TYPE_MAPPING) |
|
511 |
||
512 |
def create_temp_table(self, session, table, schema): |
|
513 |
# we don't want on commit drop, this may cause problem when |
|
514 |
# running with an ldap source, and table will be deleted manually any way |
|
515 |
# on commit |
|
516 |
sql = self.dbhelper.sql_temporary_table(table, schema, False) |
|
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
517 |
self.doexec(session, sql) |
1792 | 518 |
|
0 | 519 |
def create_eid(self, session): |
520 |
self._eid_creation_lock.acquire() |
|
521 |
try: |
|
522 |
for sql in self.dbhelper.sqls_increment_sequence('entities_id_seq'): |
|
2310
80fcdfbb8eed
oops, needs the cursor
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2306
diff
changeset
|
523 |
cursor = self.doexec(session, sql) |
0 | 524 |
return cursor.fetchone()[0] |
525 |
finally: |
|
526 |
self._eid_creation_lock.release() |
|
527 |
||
528 |
def add_info(self, session, entity, source, extid=None): |
|
529 |
"""add type and source info for an eid into the system table""" |
|
530 |
# begin by inserting eid/type/source/extid into the entities table |
|
1952
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
531 |
if extid is not None: |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
532 |
assert isinstance(extid, str) |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
533 |
extid = b64encode(extid) |
3399
2b84f4adb6f8
use __regid__ instead of id on appobject classes
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3240
diff
changeset
|
534 |
attrs = {'type': entity.__regid__, 'eid': entity.eid, 'extid': extid, |
1952
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
535 |
'source': source.uri, 'mtime': datetime.now()} |
0 | 536 |
session.system_sql(self.sqlgen.insert('entities', attrs), attrs) |
537 |
||
538 |
def delete_info(self, session, eid, etype, uri, extid): |
|
539 |
"""delete system information on deletion of an entity by transfering |
|
540 |
record from the entities table to the deleted_entities table |
|
541 |
""" |
|
542 |
attrs = {'eid': eid} |
|
543 |
session.system_sql(self.sqlgen.delete('entities', attrs), attrs) |
|
2955
6bb5025c9fc7
remove some pretty old deprecation code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2759
diff
changeset
|
544 |
if extid is not None: |
6bb5025c9fc7
remove some pretty old deprecation code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2759
diff
changeset
|
545 |
assert isinstance(extid, str), type(extid) |
6bb5025c9fc7
remove some pretty old deprecation code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2759
diff
changeset
|
546 |
extid = b64encode(extid) |
6bb5025c9fc7
remove some pretty old deprecation code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2759
diff
changeset
|
547 |
attrs = {'type': etype, 'eid': eid, 'extid': extid, |
6bb5025c9fc7
remove some pretty old deprecation code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2759
diff
changeset
|
548 |
'source': uri, 'dtime': datetime.now()} |
6bb5025c9fc7
remove some pretty old deprecation code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2759
diff
changeset
|
549 |
session.system_sql(self.sqlgen.insert('deleted_entities', attrs), attrs) |
1792 | 550 |
|
0 | 551 |
def fti_unindex_entity(self, session, eid): |
552 |
"""remove text content for entity with the given eid from the full text |
|
553 |
index |
|
554 |
""" |
|
555 |
try: |
|
556 |
self.indexer.cursor_unindex_object(eid, session.pool['system']) |
|
557 |
except: |
|
558 |
if self.indexer is not None: |
|
559 |
self.exception('error while unindexing %s', eid) |
|
1792 | 560 |
|
0 | 561 |
def fti_index_entity(self, session, entity): |
562 |
"""add text content of a created/modified entity to the full text index |
|
563 |
""" |
|
564 |
self.info('reindexing %r', entity.eid) |
|
565 |
try: |
|
566 |
self.indexer.cursor_reindex_object(entity.eid, entity, |
|
567 |
session.pool['system']) |
|
568 |
except: |
|
569 |
if self.indexer is not None: |
|
570 |
self.exception('error while reindexing %s', entity) |
|
571 |
# update entities.mtime |
|
1016
26387b836099
use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents:
973
diff
changeset
|
572 |
attrs = {'eid': entity.eid, 'mtime': datetime.now()} |
0 | 573 |
session.system_sql(self.sqlgen.update('entities', attrs, ['eid']), attrs) |
1792 | 574 |
|
0 | 575 |
def modified_entities(self, session, etypes, mtime): |
576 |
"""return a 2-uple: |
|
577 |
* list of (etype, eid) of entities of the given types which have been |
|
578 |
modified since the given timestamp (actually entities whose full text |
|
579 |
index content has changed) |
|
580 |
* list of (etype, eid) of entities of the given types which have been |
|
581 |
deleted since the given timestamp |
|
582 |
""" |
|
583 |
modsql = _modified_sql('entities', etypes) |
|
584 |
cursor = session.system_sql(modsql, {'time': mtime}) |
|
585 |
modentities = cursor.fetchall() |
|
586 |
delsql = _modified_sql('deleted_entities', etypes) |
|
587 |
cursor = session.system_sql(delsql, {'time': mtime}) |
|
588 |
delentities = cursor.fetchall() |
|
589 |
return modentities, delentities |
|
590 |
||
591 |
||
592 |
def sql_schema(driver): |
|
593 |
helper = get_adv_func_helper(driver) |
|
594 |
schema = """ |
|
595 |
/* Create the repository's system database */ |
|
596 |
||
597 |
%s |
|
598 |
||
599 |
CREATE TABLE entities ( |
|
600 |
eid INTEGER PRIMARY KEY NOT NULL, |
|
601 |
type VARCHAR(64) NOT NULL, |
|
602 |
source VARCHAR(64) NOT NULL, |
|
603 |
mtime TIMESTAMP NOT NULL, |
|
604 |
extid VARCHAR(256) |
|
605 |
); |
|
606 |
CREATE INDEX entities_type_idx ON entities(type); |
|
607 |
CREATE INDEX entities_mtime_idx ON entities(mtime); |
|
608 |
CREATE INDEX entities_extid_idx ON entities(extid); |
|
609 |
||
610 |
CREATE TABLE deleted_entities ( |
|
611 |
eid INTEGER PRIMARY KEY NOT NULL, |
|
612 |
type VARCHAR(64) NOT NULL, |
|
613 |
source VARCHAR(64) NOT NULL, |
|
614 |
dtime TIMESTAMP NOT NULL, |
|
615 |
extid VARCHAR(256) |
|
616 |
); |
|
617 |
CREATE INDEX deleted_entities_type_idx ON deleted_entities(type); |
|
618 |
CREATE INDEX deleted_entities_dtime_idx ON deleted_entities(dtime); |
|
619 |
CREATE INDEX deleted_entities_extid_idx ON deleted_entities(extid); |
|
620 |
""" % helper.sql_create_sequence('entities_id_seq') |
|
621 |
return schema |
|
622 |
||
623 |
||
624 |
def sql_drop_schema(driver): |
|
625 |
helper = get_adv_func_helper(driver) |
|
626 |
return """ |
|
627 |
%s |
|
628 |
DROP TABLE entities; |
|
629 |
DROP TABLE deleted_entities; |
|
630 |
""" % helper.sql_drop_sequence('entities_id_seq') |
|
631 |
||
632 |
||
633 |
def grant_schema(user, set_owner=True): |
|
634 |
result = '' |
|
635 |
if set_owner: |
|
636 |
result = 'ALTER TABLE entities OWNER TO %s;\n' % user |
|
637 |
result += 'ALTER TABLE deleted_entities OWNER TO %s;\n' % user |
|
638 |
result += 'ALTER TABLE entities_id_seq OWNER TO %s;\n' % user |
|
639 |
result += 'GRANT ALL ON entities TO %s;\n' % user |
|
640 |
result += 'GRANT ALL ON deleted_entities TO %s;\n' % user |
|
641 |
result += 'GRANT ALL ON entities_id_seq TO %s;\n' % user |
|
642 |
return result |