author | Sylvain Thénault <sylvain.thenault@logilab.fr> |
Tue, 21 Jul 2009 23:09:27 +0200 | |
branch | stable |
changeset 2413 | b1bea4cab594 |
parent 2354 | 9b4bac626977 |
child 2476 | 1294a6bdf3bf |
child 2566 | 714a8743d423 |
permissions | -rw-r--r-- |
0 | 1 |
"""Adapters for native cubicweb sources. |
2 |
||
1952
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
3 |
Notes: |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
4 |
* extid (aka external id, the primary key of an entity in the external source |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
5 |
from which it comes from) are stored in a varchar column encoded as a base64 |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
6 |
string. This is because it should actually be Bytes but we want an index on |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
7 |
it for fast querying. |
2056 | 8 |
|
0 | 9 |
:organization: Logilab |
1977
606923dff11b
big bunch of copyright / docstring update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1954
diff
changeset
|
10 |
:copyright: 2001-2009 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. |
0 | 11 |
:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr |
1977
606923dff11b
big bunch of copyright / docstring update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1954
diff
changeset
|
12 |
:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses |
0 | 13 |
""" |
14 |
__docformat__ = "restructuredtext en" |
|
15 |
||
16 |
from threading import Lock |
|
1016
26387b836099
use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents:
973
diff
changeset
|
17 |
from datetime import datetime |
1952
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
18 |
from base64 import b64decode, b64encode |
0 | 19 |
|
20 |
from logilab.common.cache import Cache |
|
21 |
from logilab.common.configuration import REQUIRED |
|
22 |
from logilab.common.adbh import get_adv_func_helper |
|
23 |
||
24 |
from indexer import get_indexer |
|
25 |
||
26 |
from cubicweb import UnknownEid, AuthenticationError, Binary, server |
|
27 |
from cubicweb.server.utils import crypt_password |
|
1251
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1079
diff
changeset
|
28 |
from cubicweb.server.sqlutils import SQL_PREFIX, SQLAdapterMixIn |
0 | 29 |
from cubicweb.server.rqlannotation import set_qdata |
30 |
from cubicweb.server.sources import AbstractSource |
|
31 |
from cubicweb.server.sources.rql2sql import SQLGenerator |
|
32 |
||
33 |
||
2354
9b4bac626977
ability to map attributes to something else than usual cw mapping on sql generation
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2310
diff
changeset
|
34 |
ATTR_MAP = {} |
0 | 35 |
NONSYSTEM_ETYPES = set() |
36 |
NONSYSTEM_RELATIONS = set() |
|
37 |
||
38 |
class LogCursor(object): |
|
39 |
def __init__(self, cursor): |
|
40 |
self.cu = cursor |
|
1792 | 41 |
|
0 | 42 |
def execute(self, query, args=None): |
43 |
"""Execute a query. |
|
44 |
it's a function just so that it shows up in profiling |
|
45 |
""" |
|
46 |
if server.DEBUG: |
|
47 |
print 'exec', query, args |
|
48 |
try: |
|
49 |
self.cu.execute(str(query), args) |
|
50 |
except Exception, ex: |
|
51 |
print "sql: %r\n args: %s\ndbms message: %r" % ( |
|
52 |
query, args, ex.args[0]) |
|
53 |
raise |
|
1792 | 54 |
|
0 | 55 |
def fetchall(self): |
56 |
return self.cu.fetchall() |
|
1792 | 57 |
|
0 | 58 |
def fetchone(self): |
59 |
return self.cu.fetchone() |
|
1792 | 60 |
|
0 | 61 |
def make_schema(selected, solution, table, typemap): |
62 |
"""return a sql schema to store RQL query result""" |
|
63 |
sql = [] |
|
64 |
varmap = {} |
|
65 |
for i, term in enumerate(selected): |
|
66 |
name = 'C%s' % i |
|
67 |
key = term.as_string() |
|
68 |
varmap[key] = '%s.%s' % (table, name) |
|
69 |
ttype = term.get_type(solution) |
|
70 |
try: |
|
71 |
sql.append('%s %s' % (name, typemap[ttype])) |
|
72 |
except KeyError: |
|
73 |
# assert not schema(ttype).is_final() |
|
74 |
sql.append('%s %s' % (name, typemap['Int'])) |
|
75 |
return ','.join(sql), varmap |
|
76 |
||
77 |
def _modified_sql(table, etypes): |
|
78 |
# XXX protect against sql injection |
|
79 |
if len(etypes) > 1: |
|
80 |
restr = 'type IN (%s)' % ','.join("'%s'" % etype for etype in etypes) |
|
81 |
else: |
|
82 |
restr = "type='%s'" % etypes[0] |
|
83 |
if table == 'entities': |
|
84 |
attr = 'mtime' |
|
85 |
else: |
|
86 |
attr = 'dtime' |
|
87 |
return 'SELECT type, eid FROM %s WHERE %s AND %s > %%(time)s' % ( |
|
88 |
table, restr, attr) |
|
89 |
||
90 |
||
91 |
class NativeSQLSource(SQLAdapterMixIn, AbstractSource): |
|
92 |
"""adapter for source using the native cubicweb schema (see below) |
|
93 |
""" |
|
2354
9b4bac626977
ability to map attributes to something else than usual cw mapping on sql generation
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2310
diff
changeset
|
94 |
sqlgen_class = SQLGenerator |
0 | 95 |
# need default value on class since migration doesn't call init method |
96 |
has_deleted_entitites_table = True |
|
1792 | 97 |
|
1398
5fe84a5f7035
rename internal entity types to have CW prefix instead of E
sylvain.thenault@logilab.fr
parents:
1263
diff
changeset
|
98 |
passwd_rql = "Any P WHERE X is CWUser, X login %(login)s, X upassword P" |
5fe84a5f7035
rename internal entity types to have CW prefix instead of E
sylvain.thenault@logilab.fr
parents:
1263
diff
changeset
|
99 |
auth_rql = "Any X WHERE X is CWUser, X login %(login)s, X upassword %(pwd)s" |
5fe84a5f7035
rename internal entity types to have CW prefix instead of E
sylvain.thenault@logilab.fr
parents:
1263
diff
changeset
|
100 |
_sols = ({'X': 'CWUser', 'P': 'Password'},) |
1792 | 101 |
|
0 | 102 |
options = ( |
103 |
('db-driver', |
|
104 |
{'type' : 'string', |
|
105 |
'default': 'postgres', |
|
106 |
'help': 'database driver (postgres or sqlite)', |
|
107 |
'group': 'native-source', 'inputlevel': 1, |
|
108 |
}), |
|
109 |
('db-host', |
|
110 |
{'type' : 'string', |
|
111 |
'default': '', |
|
112 |
'help': 'database host', |
|
113 |
'group': 'native-source', 'inputlevel': 1, |
|
114 |
}), |
|
115 |
('db-name', |
|
116 |
{'type' : 'string', |
|
117 |
'default': REQUIRED, |
|
118 |
'help': 'database name', |
|
119 |
'group': 'native-source', 'inputlevel': 0, |
|
120 |
}), |
|
121 |
('db-user', |
|
122 |
{'type' : 'string', |
|
123 |
'default': 'cubicweb', |
|
124 |
'help': 'database user', |
|
125 |
'group': 'native-source', 'inputlevel': 0, |
|
126 |
}), |
|
127 |
('db-password', |
|
128 |
{'type' : 'password', |
|
129 |
'default': '', |
|
130 |
'help': 'database password', |
|
131 |
'group': 'native-source', 'inputlevel': 0, |
|
132 |
}), |
|
133 |
('db-encoding', |
|
134 |
{'type' : 'string', |
|
135 |
'default': 'utf8', |
|
136 |
'help': 'database encoding', |
|
137 |
'group': 'native-source', 'inputlevel': 1, |
|
138 |
}), |
|
139 |
) |
|
1792 | 140 |
|
0 | 141 |
def __init__(self, repo, appschema, source_config, *args, **kwargs): |
142 |
SQLAdapterMixIn.__init__(self, source_config) |
|
143 |
AbstractSource.__init__(self, repo, appschema, source_config, |
|
144 |
*args, **kwargs) |
|
145 |
# sql generator |
|
2354
9b4bac626977
ability to map attributes to something else than usual cw mapping on sql generation
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2310
diff
changeset
|
146 |
self._rql_sqlgen = self.sqlgen_class(appschema, self.dbhelper, |
9b4bac626977
ability to map attributes to something else than usual cw mapping on sql generation
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2310
diff
changeset
|
147 |
self.encoding, ATTR_MAP.copy()) |
0 | 148 |
# full text index helper |
149 |
self.indexer = get_indexer(self.dbdriver, self.encoding) |
|
150 |
# advanced functionality helper |
|
151 |
self.dbhelper.fti_uid_attr = self.indexer.uid_attr |
|
152 |
self.dbhelper.fti_table = self.indexer.table |
|
153 |
self.dbhelper.fti_restriction_sql = self.indexer.restriction_sql |
|
154 |
self.dbhelper.fti_need_distinct_query = self.indexer.need_distinct |
|
155 |
# sql queries cache |
|
156 |
self._cache = Cache(repo.config['rql-cache-size']) |
|
157 |
self._temp_table_data = {} |
|
158 |
self._eid_creation_lock = Lock() |
|
2072
8008e8812d76
deactivate sqlite connection wrapping for unittest_multisources for now
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2066
diff
changeset
|
159 |
# XXX no_sqlite_wrap trick since we've a sqlite locking pb when |
8008e8812d76
deactivate sqlite connection wrapping for unittest_multisources for now
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2066
diff
changeset
|
160 |
# running unittest_multisources with the wrapping below |
8008e8812d76
deactivate sqlite connection wrapping for unittest_multisources for now
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2066
diff
changeset
|
161 |
if self.dbdriver == 'sqlite' and \ |
8008e8812d76
deactivate sqlite connection wrapping for unittest_multisources for now
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2066
diff
changeset
|
162 |
not getattr(repo.config, 'no_sqlite_wrap', False): |
2064
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
163 |
from cubicweb.server.sources.extlite import ConnectionWrapper |
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
164 |
self.get_connection = lambda: ConnectionWrapper(self) |
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
165 |
self.check_connection = lambda cnx: cnx |
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
166 |
def pool_reset(cnx): |
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
167 |
if cnx._cnx is not None: |
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
168 |
cnx._cnx.close() |
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
169 |
cnx._cnx = None |
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
170 |
self.pool_reset = pool_reset |
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
171 |
|
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
172 |
@property |
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
173 |
def _sqlcnx(self): |
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
174 |
# XXX: sqlite connections can only be used in the same thread, so |
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
175 |
# create a new one each time necessary. If it appears to be time |
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
176 |
# consuming, find another way |
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
177 |
return SQLAdapterMixIn.get_connection(self) |
0 | 178 |
|
179 |
def reset_caches(self): |
|
180 |
"""method called during test to reset potential source caches""" |
|
181 |
self._cache = Cache(self.repo.config['rql-cache-size']) |
|
1792 | 182 |
|
0 | 183 |
def clear_eid_cache(self, eid, etype): |
184 |
"""clear potential caches for the given eid""" |
|
185 |
self._cache.pop('%s X WHERE X eid %s' % (etype, eid), None) |
|
186 |
self._cache.pop('Any X WHERE X eid %s' % eid, None) |
|
1792 | 187 |
|
0 | 188 |
def sqlexec(self, session, sql, args=None): |
189 |
"""execute the query and return its result""" |
|
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
190 |
return self.process_result(self.doexec(session, sql, args)) |
1792 | 191 |
|
0 | 192 |
def init_creating(self): |
2064
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
193 |
pool = self.repo._get_pool() |
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
194 |
pool.pool_set() |
0 | 195 |
# check full text index availibility |
196 |
if not self.indexer.has_fti_table(pool['system']): |
|
197 |
self.error('no text index table') |
|
198 |
self.indexer = None |
|
2064
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
199 |
pool.pool_reset() |
0 | 200 |
self.repo._free_pool(pool) |
201 |
||
202 |
def init(self): |
|
1792 | 203 |
self.init_creating() |
0 | 204 |
pool = self.repo._get_pool() |
2064
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
205 |
pool.pool_set() |
0 | 206 |
# XXX cubicweb < 2.42 compat |
207 |
if 'deleted_entities' in self.dbhelper.list_tables(pool['system']): |
|
208 |
self.has_deleted_entitites_table = True |
|
209 |
else: |
|
210 |
self.has_deleted_entitites_table = False |
|
2064
a5cd3a92314a
properly call [re]set_pool, fix connection handling so we have a change to get cw running on top of a sqlite dabase with threads activated
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2056
diff
changeset
|
211 |
pool.pool_reset() |
0 | 212 |
self.repo._free_pool(pool) |
1792 | 213 |
|
2354
9b4bac626977
ability to map attributes to something else than usual cw mapping on sql generation
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2310
diff
changeset
|
214 |
def map_attribute(self, etype, attr, cb): |
9b4bac626977
ability to map attributes to something else than usual cw mapping on sql generation
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2310
diff
changeset
|
215 |
self._rql_sqlgen.attr_map['%s.%s' % (etype, attr)] = cb |
9b4bac626977
ability to map attributes to something else than usual cw mapping on sql generation
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2310
diff
changeset
|
216 |
|
0 | 217 |
# ISource interface ####################################################### |
218 |
||
219 |
def compile_rql(self, rql): |
|
220 |
rqlst = self.repo.querier._rqlhelper.parse(rql) |
|
221 |
rqlst.restricted_vars = () |
|
222 |
rqlst.children[0].solutions = self._sols |
|
223 |
self.repo.querier.sqlgen_annotate(rqlst) |
|
438 | 224 |
set_qdata(self.schema.rschema, rqlst, ()) |
0 | 225 |
return rqlst |
1792 | 226 |
|
0 | 227 |
def set_schema(self, schema): |
228 |
"""set the application'schema""" |
|
229 |
self._cache = Cache(self.repo.config['rql-cache-size']) |
|
230 |
self.cache_hit, self.cache_miss, self.no_cache = 0, 0, 0 |
|
231 |
self.schema = schema |
|
232 |
try: |
|
233 |
self._rql_sqlgen.schema = schema |
|
234 |
except AttributeError: |
|
235 |
pass # __init__ |
|
1398
5fe84a5f7035
rename internal entity types to have CW prefix instead of E
sylvain.thenault@logilab.fr
parents:
1263
diff
changeset
|
236 |
if 'CWUser' in schema: # probably an empty schema if not true... |
0 | 237 |
# rql syntax trees used to authenticate users |
238 |
self._passwd_rqlst = self.compile_rql(self.passwd_rql) |
|
239 |
self._auth_rqlst = self.compile_rql(self.auth_rql) |
|
1792 | 240 |
|
0 | 241 |
def support_entity(self, etype, write=False): |
242 |
"""return true if the given entity's type is handled by this adapter |
|
243 |
if write is true, return true only if it's a RW support |
|
244 |
""" |
|
245 |
return not etype in NONSYSTEM_ETYPES |
|
1792 | 246 |
|
0 | 247 |
def support_relation(self, rtype, write=False): |
248 |
"""return true if the given relation's type is handled by this adapter |
|
249 |
if write is true, return true only if it's a RW support |
|
250 |
""" |
|
251 |
if write: |
|
252 |
return not rtype in NONSYSTEM_RELATIONS |
|
253 |
# due to current multi-sources implementation, the system source |
|
1792 | 254 |
# can't claim not supporting a relation |
0 | 255 |
return True #not rtype == 'content_for' |
256 |
||
257 |
def authenticate(self, session, login, password): |
|
1398
5fe84a5f7035
rename internal entity types to have CW prefix instead of E
sylvain.thenault@logilab.fr
parents:
1263
diff
changeset
|
258 |
"""return CWUser eid for the given login/password if this account is |
0 | 259 |
defined in this source, else raise `AuthenticationError` |
260 |
||
261 |
two queries are needed since passwords are stored crypted, so we have |
|
262 |
to fetch the salt first |
|
263 |
""" |
|
264 |
args = {'login': login, 'pwd' : password} |
|
265 |
if password is not None: |
|
266 |
rset = self.syntax_tree_search(session, self._passwd_rqlst, args) |
|
267 |
try: |
|
268 |
pwd = rset[0][0] |
|
269 |
except IndexError: |
|
270 |
raise AuthenticationError('bad login') |
|
1954 | 271 |
# passwords are stored using the Bytes type, so we get a StringIO |
0 | 272 |
if pwd is not None: |
273 |
args['pwd'] = crypt_password(password, pwd.getvalue()[:2]) |
|
274 |
# get eid from login and (crypted) password |
|
275 |
rset = self.syntax_tree_search(session, self._auth_rqlst, args) |
|
276 |
try: |
|
277 |
return rset[0][0] |
|
278 |
except IndexError: |
|
279 |
raise AuthenticationError('bad password') |
|
1792 | 280 |
|
281 |
def syntax_tree_search(self, session, union, args=None, cachekey=None, |
|
0 | 282 |
varmap=None): |
283 |
"""return result from this source for a rql query (actually from |
|
284 |
a rql syntax tree and a solution dictionary mapping each used |
|
285 |
variable to a possible type). If cachekey is given, the query |
|
286 |
necessary to fetch the results (but not the results themselves) |
|
287 |
may be cached using this key. |
|
288 |
""" |
|
289 |
if server.DEBUG: |
|
290 |
print 'RQL FOR NATIVE SOURCE', self.uri, cachekey |
|
291 |
if varmap: |
|
292 |
print 'USING VARMAP', varmap |
|
293 |
print union.as_string() |
|
294 |
if args: print 'ARGS', args |
|
295 |
print 'SOLUTIONS', ','.join(str(s.solutions) for s in union.children) |
|
296 |
# remember number of actually selected term (sql generation may append some) |
|
297 |
if cachekey is None: |
|
298 |
self.no_cache += 1 |
|
299 |
# generate sql query if we are able to do so (not supported types...) |
|
300 |
sql, query_args = self._rql_sqlgen.generate(union, args, varmap) |
|
301 |
else: |
|
302 |
# sql may be cached |
|
303 |
try: |
|
304 |
sql, query_args = self._cache[cachekey] |
|
305 |
self.cache_hit += 1 |
|
306 |
except KeyError: |
|
307 |
self.cache_miss += 1 |
|
308 |
sql, query_args = self._rql_sqlgen.generate(union, args, varmap) |
|
309 |
self._cache[cachekey] = sql, query_args |
|
310 |
args = self.merge_args(args, query_args) |
|
311 |
assert isinstance(sql, basestring), repr(sql) |
|
312 |
try: |
|
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
313 |
cursor = self.doexec(session, sql, args) |
0 | 314 |
except (self.dbapi_module.OperationalError, |
315 |
self.dbapi_module.InterfaceError): |
|
316 |
# FIXME: better detection of deconnection pb |
|
317 |
self.info("request failed '%s' ... retry with a new cursor", sql) |
|
318 |
session.pool.reconnect(self) |
|
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
319 |
cursor = self.doexec(session, sql, args) |
0 | 320 |
res = self.process_result(cursor) |
321 |
if server.DEBUG: |
|
322 |
print '------>', res |
|
323 |
return res |
|
1792 | 324 |
|
0 | 325 |
def flying_insert(self, table, session, union, args=None, varmap=None): |
326 |
"""similar as .syntax_tree_search, but inserts data in the |
|
327 |
temporary table (on-the-fly if possible, eg for the system |
|
328 |
source whose the given cursor come from). If not possible, |
|
329 |
inserts all data by calling .executemany(). |
|
330 |
""" |
|
331 |
if self.uri == 'system': |
|
332 |
if server.DEBUG: |
|
333 |
print 'FLYING RQL FOR SOURCE', self.uri |
|
334 |
if varmap: |
|
335 |
print 'USING VARMAP', varmap |
|
336 |
print union.as_string() |
|
337 |
print 'SOLUTIONS', ','.join(str(s.solutions) for s in union.children) |
|
338 |
# generate sql queries if we are able to do so |
|
339 |
sql, query_args = self._rql_sqlgen.generate(union, args, varmap) |
|
340 |
query = 'INSERT INTO %s %s' % (table, sql.encode(self.encoding)) |
|
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
341 |
self.doexec(session, query, self.merge_args(args, query_args)) |
0 | 342 |
else: |
343 |
super(NativeSQLSource, self).flying_insert(table, session, union, |
|
344 |
args, varmap) |
|
1792 | 345 |
|
0 | 346 |
def _manual_insert(self, results, table, session): |
347 |
"""insert given result into a temporary table on the system source""" |
|
348 |
if not results: |
|
349 |
return |
|
350 |
query_args = ['%%(%s)s' % i for i in xrange(len(results[0]))] |
|
351 |
query = 'INSERT INTO %s VALUES(%s)' % (table, ','.join(query_args)) |
|
352 |
kwargs_list = [] |
|
353 |
for row in results: |
|
354 |
kwargs = {} |
|
355 |
row = tuple(row) |
|
356 |
for index, cell in enumerate(row): |
|
2066
2c4bf4ee88a2
cleanup, stop encoding unicode string in manual_insert, no more necessary and make crash recent sqlite w/ 8bit string
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2064
diff
changeset
|
357 |
if isinstance(cell, Binary): |
0 | 358 |
cell = self.binary(cell.getvalue()) |
359 |
kwargs[str(index)] = cell |
|
360 |
kwargs_list.append(kwargs) |
|
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
361 |
self.doexecmany(session, query, kwargs_list) |
0 | 362 |
|
363 |
def clean_temp_data(self, session, temptables): |
|
364 |
"""remove temporary data, usually associated to temporary tables""" |
|
365 |
if temptables: |
|
366 |
for table in temptables: |
|
367 |
try: |
|
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
368 |
self.doexec(session,'DROP TABLE %s' % table) |
0 | 369 |
except: |
370 |
pass |
|
371 |
try: |
|
372 |
del self._temp_table_data[table] |
|
373 |
except KeyError: |
|
374 |
continue |
|
1792 | 375 |
|
0 | 376 |
def add_entity(self, session, entity): |
377 |
"""add a new entity to the source""" |
|
378 |
attrs = self.preprocess_entity(entity) |
|
1251
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1079
diff
changeset
|
379 |
sql = self.sqlgen.insert(SQL_PREFIX + str(entity.e_schema), attrs) |
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
380 |
self.doexec(session, sql, attrs) |
1792 | 381 |
|
0 | 382 |
def update_entity(self, session, entity): |
383 |
"""replace an entity in the source""" |
|
384 |
attrs = self.preprocess_entity(entity) |
|
1251
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1079
diff
changeset
|
385 |
sql = self.sqlgen.update(SQL_PREFIX + str(entity.e_schema), attrs, [SQL_PREFIX + 'eid']) |
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
386 |
self.doexec(session, sql, attrs) |
0 | 387 |
|
388 |
def delete_entity(self, session, etype, eid): |
|
389 |
"""delete an entity from the source""" |
|
1251
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1079
diff
changeset
|
390 |
attrs = {SQL_PREFIX + 'eid': eid} |
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1079
diff
changeset
|
391 |
sql = self.sqlgen.delete(SQL_PREFIX + etype, attrs) |
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
392 |
self.doexec(session, sql, attrs) |
0 | 393 |
|
394 |
def add_relation(self, session, subject, rtype, object): |
|
395 |
"""add a relation to the source""" |
|
396 |
attrs = {'eid_from': subject, 'eid_to': object} |
|
397 |
sql = self.sqlgen.insert('%s_relation' % rtype, attrs) |
|
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
398 |
self.doexec(session, sql, attrs) |
1792 | 399 |
|
0 | 400 |
def delete_relation(self, session, subject, rtype, object): |
401 |
"""delete a relation from the source""" |
|
402 |
rschema = self.schema.rschema(rtype) |
|
403 |
if rschema.inlined: |
|
1251
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1079
diff
changeset
|
404 |
table = SQL_PREFIX + session.describe(subject)[0] |
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1079
diff
changeset
|
405 |
column = SQL_PREFIX + rtype |
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1079
diff
changeset
|
406 |
sql = 'UPDATE %s SET %s=NULL WHERE %seid=%%(eid)s' % (table, column, |
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1079
diff
changeset
|
407 |
SQL_PREFIX) |
0 | 408 |
attrs = {'eid' : subject} |
409 |
else: |
|
410 |
attrs = {'eid_from': subject, 'eid_to': object} |
|
411 |
sql = self.sqlgen.delete('%s_relation' % rtype, attrs) |
|
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
412 |
self.doexec(session, sql, attrs) |
0 | 413 |
|
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
414 |
def doexec(self, session, query, args=None): |
0 | 415 |
"""Execute a query. |
416 |
it's a function just so that it shows up in profiling |
|
417 |
""" |
|
418 |
if server.DEBUG: |
|
419 |
print 'exec', query, args |
|
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
420 |
cursor = session.pool[self.uri] |
0 | 421 |
try: |
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
422 |
# str(query) to avoid error if it's an unicode string |
0 | 423 |
cursor.execute(str(query), args) |
424 |
except Exception, ex: |
|
425 |
self.critical("sql: %r\n args: %s\ndbms message: %r", |
|
426 |
query, args, ex.args[0]) |
|
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
427 |
try: |
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
428 |
session.pool.connection(self.uri).rollback() |
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
429 |
self.critical('transaction has been rollbacked') |
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
430 |
except: |
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
431 |
pass |
0 | 432 |
raise |
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
433 |
return cursor |
1792 | 434 |
|
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
435 |
def doexecmany(self, session, query, args): |
0 | 436 |
"""Execute a query. |
437 |
it's a function just so that it shows up in profiling |
|
438 |
""" |
|
439 |
if server.DEBUG: |
|
440 |
print 'execmany', query, 'with', len(args), 'arguments' |
|
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
441 |
cursor = session.pool[self.uri] |
0 | 442 |
try: |
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
443 |
# str(query) to avoid error if it's an unicode string |
0 | 444 |
cursor.executemany(str(query), args) |
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
445 |
except Exception, ex: |
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
446 |
self.critical("sql many: %r\n args: %s\ndbms message: %r", |
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
447 |
query, args, ex.args[0]) |
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
448 |
try: |
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
449 |
session.pool.connection(self.uri).rollback() |
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
450 |
self.critical('transaction has been rollbacked') |
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
451 |
except: |
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
452 |
pass |
0 | 453 |
raise |
1792 | 454 |
|
0 | 455 |
# short cut to method requiring advanced db helper usage ################## |
1792 | 456 |
|
0 | 457 |
def create_index(self, session, table, column, unique=False): |
458 |
cursor = LogCursor(session.pool[self.uri]) |
|
459 |
self.dbhelper.create_index(cursor, table, column, unique) |
|
1792 | 460 |
|
0 | 461 |
def drop_index(self, session, table, column, unique=False): |
462 |
cursor = LogCursor(session.pool[self.uri]) |
|
463 |
self.dbhelper.drop_index(cursor, table, column, unique) |
|
464 |
||
465 |
# system source interface ################################################# |
|
466 |
||
467 |
def eid_type_source(self, session, eid): |
|
468 |
"""return a tuple (type, source, extid) for the entity with id <eid>""" |
|
469 |
sql = 'SELECT type, source, extid FROM entities WHERE eid=%s' % eid |
|
470 |
try: |
|
471 |
res = session.system_sql(sql).fetchone() |
|
472 |
except: |
|
1079
452cb76fe07a
backport typo fix
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
973
diff
changeset
|
473 |
assert session.pool, 'session has no pool set' |
0 | 474 |
raise UnknownEid(eid) |
475 |
if res is None: |
|
476 |
raise UnknownEid(eid) |
|
1952
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
477 |
if res[-1] is not None: |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
478 |
if not isinstance(res, list): |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
479 |
res = list(res) |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
480 |
res[-1] = b64decode(res[-1]) |
0 | 481 |
return res |
482 |
||
1952
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
483 |
def extid2eid(self, session, source, extid): |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
484 |
"""get eid from an external id. Return None if no record found.""" |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
485 |
assert isinstance(extid, str) |
0 | 486 |
cursor = session.system_sql('SELECT eid FROM entities WHERE ' |
487 |
'extid=%(x)s AND source=%(s)s', |
|
1952
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
488 |
{'x': b64encode(extid), 's': source.uri}) |
0 | 489 |
# XXX testing rowcount cause strange bug with sqlite, results are there |
490 |
# but rowcount is 0 |
|
1792 | 491 |
#if cursor.rowcount > 0: |
0 | 492 |
try: |
493 |
result = cursor.fetchone() |
|
494 |
if result: |
|
1954 | 495 |
return result[0] |
0 | 496 |
except: |
497 |
pass |
|
498 |
return None |
|
1792 | 499 |
|
0 | 500 |
def temp_table_def(self, selected, sol, table): |
501 |
return make_schema(selected, sol, table, self.dbhelper.TYPE_MAPPING) |
|
502 |
||
503 |
def create_temp_table(self, session, table, schema): |
|
504 |
# we don't want on commit drop, this may cause problem when |
|
505 |
# running with an ldap source, and table will be deleted manually any way |
|
506 |
# on commit |
|
507 |
sql = self.dbhelper.sql_temporary_table(table, schema, False) |
|
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
508 |
self.doexec(session, sql) |
1792 | 509 |
|
0 | 510 |
def create_eid(self, session): |
511 |
self._eid_creation_lock.acquire() |
|
512 |
try: |
|
513 |
for sql in self.dbhelper.sqls_increment_sequence('entities_id_seq'): |
|
2310
80fcdfbb8eed
oops, needs the cursor
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2306
diff
changeset
|
514 |
cursor = self.doexec(session, sql) |
0 | 515 |
return cursor.fetchone()[0] |
516 |
finally: |
|
517 |
self._eid_creation_lock.release() |
|
518 |
||
519 |
def add_info(self, session, entity, source, extid=None): |
|
520 |
"""add type and source info for an eid into the system table""" |
|
521 |
# begin by inserting eid/type/source/extid into the entities table |
|
1952
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
522 |
if extid is not None: |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
523 |
assert isinstance(extid, str) |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
524 |
extid = b64encode(extid) |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
525 |
attrs = {'type': entity.id, 'eid': entity.eid, 'extid': extid, |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
526 |
'source': source.uri, 'mtime': datetime.now()} |
0 | 527 |
session.system_sql(self.sqlgen.insert('entities', attrs), attrs) |
528 |
||
529 |
def delete_info(self, session, eid, etype, uri, extid): |
|
530 |
"""delete system information on deletion of an entity by transfering |
|
531 |
record from the entities table to the deleted_entities table |
|
532 |
""" |
|
533 |
attrs = {'eid': eid} |
|
534 |
session.system_sql(self.sqlgen.delete('entities', attrs), attrs) |
|
535 |
if self.has_deleted_entitites_table: |
|
1952
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
536 |
if extid is not None: |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
537 |
assert isinstance(extid, str), type(extid) |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
538 |
extid = b64encode(extid) |
0 | 539 |
attrs = {'type': etype, 'eid': eid, 'extid': extid, |
1016
26387b836099
use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents:
973
diff
changeset
|
540 |
'source': uri, 'dtime': datetime.now()} |
0 | 541 |
session.system_sql(self.sqlgen.insert('deleted_entities', attrs), attrs) |
1792 | 542 |
|
0 | 543 |
def fti_unindex_entity(self, session, eid): |
544 |
"""remove text content for entity with the given eid from the full text |
|
545 |
index |
|
546 |
""" |
|
547 |
try: |
|
548 |
self.indexer.cursor_unindex_object(eid, session.pool['system']) |
|
549 |
except: |
|
550 |
if self.indexer is not None: |
|
551 |
self.exception('error while unindexing %s', eid) |
|
1792 | 552 |
|
0 | 553 |
def fti_index_entity(self, session, entity): |
554 |
"""add text content of a created/modified entity to the full text index |
|
555 |
""" |
|
556 |
self.info('reindexing %r', entity.eid) |
|
557 |
try: |
|
558 |
self.indexer.cursor_reindex_object(entity.eid, entity, |
|
559 |
session.pool['system']) |
|
560 |
except: |
|
561 |
if self.indexer is not None: |
|
562 |
self.exception('error while reindexing %s', entity) |
|
563 |
# update entities.mtime |
|
1016
26387b836099
use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents:
973
diff
changeset
|
564 |
attrs = {'eid': entity.eid, 'mtime': datetime.now()} |
0 | 565 |
session.system_sql(self.sqlgen.update('entities', attrs, ['eid']), attrs) |
1792 | 566 |
|
0 | 567 |
def modified_entities(self, session, etypes, mtime): |
568 |
"""return a 2-uple: |
|
569 |
* list of (etype, eid) of entities of the given types which have been |
|
570 |
modified since the given timestamp (actually entities whose full text |
|
571 |
index content has changed) |
|
572 |
* list of (etype, eid) of entities of the given types which have been |
|
573 |
deleted since the given timestamp |
|
574 |
""" |
|
575 |
modsql = _modified_sql('entities', etypes) |
|
576 |
cursor = session.system_sql(modsql, {'time': mtime}) |
|
577 |
modentities = cursor.fetchall() |
|
578 |
delsql = _modified_sql('deleted_entities', etypes) |
|
579 |
cursor = session.system_sql(delsql, {'time': mtime}) |
|
580 |
delentities = cursor.fetchall() |
|
581 |
return modentities, delentities |
|
582 |
||
583 |
||
584 |
def sql_schema(driver): |
|
585 |
helper = get_adv_func_helper(driver) |
|
586 |
schema = """ |
|
587 |
/* Create the repository's system database */ |
|
588 |
||
589 |
%s |
|
590 |
||
591 |
CREATE TABLE entities ( |
|
592 |
eid INTEGER PRIMARY KEY NOT NULL, |
|
593 |
type VARCHAR(64) NOT NULL, |
|
594 |
source VARCHAR(64) NOT NULL, |
|
595 |
mtime TIMESTAMP NOT NULL, |
|
596 |
extid VARCHAR(256) |
|
597 |
); |
|
598 |
CREATE INDEX entities_type_idx ON entities(type); |
|
599 |
CREATE INDEX entities_mtime_idx ON entities(mtime); |
|
600 |
CREATE INDEX entities_extid_idx ON entities(extid); |
|
601 |
||
602 |
CREATE TABLE deleted_entities ( |
|
603 |
eid INTEGER PRIMARY KEY NOT NULL, |
|
604 |
type VARCHAR(64) NOT NULL, |
|
605 |
source VARCHAR(64) NOT NULL, |
|
606 |
dtime TIMESTAMP NOT NULL, |
|
607 |
extid VARCHAR(256) |
|
608 |
); |
|
609 |
CREATE INDEX deleted_entities_type_idx ON deleted_entities(type); |
|
610 |
CREATE INDEX deleted_entities_dtime_idx ON deleted_entities(dtime); |
|
611 |
CREATE INDEX deleted_entities_extid_idx ON deleted_entities(extid); |
|
612 |
""" % helper.sql_create_sequence('entities_id_seq') |
|
613 |
return schema |
|
614 |
||
615 |
||
616 |
def sql_drop_schema(driver): |
|
617 |
helper = get_adv_func_helper(driver) |
|
618 |
return """ |
|
619 |
%s |
|
620 |
DROP TABLE entities; |
|
621 |
DROP TABLE deleted_entities; |
|
622 |
""" % helper.sql_drop_sequence('entities_id_seq') |
|
623 |
||
624 |
||
625 |
def grant_schema(user, set_owner=True): |
|
626 |
result = '' |
|
627 |
if set_owner: |
|
628 |
result = 'ALTER TABLE entities OWNER TO %s;\n' % user |
|
629 |
result += 'ALTER TABLE deleted_entities OWNER TO %s;\n' % user |
|
630 |
result += 'ALTER TABLE entities_id_seq OWNER TO %s;\n' % user |
|
631 |
result += 'GRANT ALL ON entities TO %s;\n' % user |
|
632 |
result += 'GRANT ALL ON deleted_entities TO %s;\n' % user |
|
633 |
result += 'GRANT ALL ON entities_id_seq TO %s;\n' % user |
|
634 |
return result |