|
1 # copyright 2003-2014 LOGILAB S.A. (Paris, FRANCE), all rights reserved. |
|
2 # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr |
|
3 # |
|
4 # This file is part of CubicWeb. |
|
5 # |
|
6 # CubicWeb is free software: you can redistribute it and/or modify it under the |
|
7 # terms of the GNU Lesser General Public License as published by the Free |
|
8 # Software Foundation, either version 2.1 of the License, or (at your option) |
|
9 # any later version. |
|
10 # |
|
11 # CubicWeb is distributed in the hope that it will be useful, but WITHOUT |
|
12 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS |
|
13 # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more |
|
14 # details. |
|
15 # |
|
16 # You should have received a copy of the GNU Lesser General Public License along |
|
17 # with CubicWeb. If not, see <http://www.gnu.org/licenses/>. |
|
18 """cubicweb server sources support""" |
|
19 from __future__ import print_function |
|
20 |
|
21 __docformat__ = "restructuredtext en" |
|
22 |
|
23 from time import time |
|
24 from logging import getLogger |
|
25 from base64 import b64decode |
|
26 |
|
27 from six import text_type |
|
28 |
|
29 from logilab.common import configuration |
|
30 from logilab.common.textutils import unormalize |
|
31 from logilab.common.deprecation import deprecated |
|
32 |
|
33 from yams.schema import role_name |
|
34 |
|
35 from cubicweb import ValidationError, set_log_methods, server |
|
36 from cubicweb.server import SOURCE_TYPES |
|
37 from cubicweb.server.edition import EditedEntity |
|
38 |
|
39 |
|
40 def dbg_st_search(uri, union, varmap, args, cachekey=None, prefix='rql for'): |
|
41 if server.DEBUG & server.DBG_RQL: |
|
42 global t |
|
43 print(' %s %s source: %s' % (prefix, uri, repr(union.as_string()))) |
|
44 t = time() |
|
45 if varmap: |
|
46 print(' using varmap', varmap) |
|
47 if server.DEBUG & server.DBG_MORE: |
|
48 print(' args', repr(args)) |
|
49 print(' cache key', cachekey) |
|
50 print(' solutions', ','.join(str(s.solutions) |
|
51 for s in union.children)) |
|
52 # return true so it can be used as assertion (and so be killed by python -O) |
|
53 return True |
|
54 |
|
55 def dbg_results(results): |
|
56 if server.DEBUG & server.DBG_RQL: |
|
57 if len(results) > 10: |
|
58 print(' -->', results[:10], '...', len(results), end=' ') |
|
59 else: |
|
60 print(' -->', results, end=' ') |
|
61 print('time: ', time() - t) |
|
62 # return true so it can be used as assertion (and so be killed by python -O) |
|
63 return True |
|
64 |
|
65 |
|
66 class AbstractSource(object): |
|
67 """an abstract class for sources""" |
|
68 |
|
69 # boolean telling if modification hooks should be called when something is |
|
70 # modified in this source |
|
71 should_call_hooks = True |
|
72 # boolean telling if the repository should connect to this source during |
|
73 # migration |
|
74 connect_for_migration = True |
|
75 |
|
76 # mappings telling which entities and relations are available in the source |
|
77 # keys are supported entity/relation types and values are boolean indicating |
|
78 # wether the support is read-only (False) or read-write (True) |
|
79 support_entities = {} |
|
80 support_relations = {} |
|
81 # a global identifier for this source, which has to be set by the source |
|
82 # instance |
|
83 uri = None |
|
84 # a reference to the system information helper |
|
85 repo = None |
|
86 # a reference to the instance'schema (may differs from the source'schema) |
|
87 schema = None |
|
88 |
|
89 # force deactivation (configuration error for instance) |
|
90 disabled = False |
|
91 |
|
92 # boolean telling if cwuri of entities from this source is the url that |
|
93 # should be used as entity's absolute url |
|
94 use_cwuri_as_url = False |
|
95 |
|
96 # source configuration options |
|
97 options = () |
|
98 |
|
99 # these are overridden by set_log_methods below |
|
100 # only defining here to prevent pylint from complaining |
|
101 info = warning = error = critical = exception = debug = lambda msg,*a,**kw: None |
|
102 |
|
103 def __init__(self, repo, source_config, eid=None): |
|
104 self.repo = repo |
|
105 self.set_schema(repo.schema) |
|
106 self.support_relations['identity'] = False |
|
107 self.eid = eid |
|
108 self.public_config = source_config.copy() |
|
109 self.public_config['use-cwuri-as-url'] = self.use_cwuri_as_url |
|
110 self.remove_sensitive_information(self.public_config) |
|
111 self.uri = source_config.pop('uri') |
|
112 # unormalize to avoid non-ascii characters in logger's name, this will cause decoding error |
|
113 # on logging |
|
114 set_log_methods(self, getLogger('cubicweb.sources.' + unormalize(text_type(self.uri)))) |
|
115 source_config.pop('type') |
|
116 self.update_config(None, self.check_conf_dict(eid, source_config, |
|
117 fail_if_unknown=False)) |
|
118 |
|
119 def __repr__(self): |
|
120 return '<%s %s source %s @%#x>' % (self.uri, self.__class__.__name__, |
|
121 self.eid, id(self)) |
|
122 |
|
123 def __lt__(self, other): |
|
124 """simple comparison function to get predictable source order, with the |
|
125 system source at last |
|
126 """ |
|
127 if self.uri == other.uri: |
|
128 return False |
|
129 if self.uri == 'system': |
|
130 return False |
|
131 if other.uri == 'system': |
|
132 return True |
|
133 return self.uri < other.uri |
|
134 |
|
135 def __eq__(self, other): |
|
136 return self.uri == other.uri |
|
137 |
|
138 def __ne__(self, other): |
|
139 return not (self == other) |
|
140 |
|
141 def backup(self, backupfile, confirm, format='native'): |
|
142 """method called to create a backup of source's data""" |
|
143 pass |
|
144 |
|
145 def restore(self, backupfile, confirm, drop, format='native'): |
|
146 """method called to restore a backup of source's data""" |
|
147 pass |
|
148 |
|
149 @classmethod |
|
150 def check_conf_dict(cls, eid, confdict, _=text_type, fail_if_unknown=True): |
|
151 """check configuration of source entity. Return config dict properly |
|
152 typed with defaults set. |
|
153 """ |
|
154 processed = {} |
|
155 for optname, optdict in cls.options: |
|
156 value = confdict.pop(optname, optdict.get('default')) |
|
157 if value is configuration.REQUIRED: |
|
158 if not fail_if_unknown: |
|
159 continue |
|
160 msg = _('specifying %s is mandatory' % optname) |
|
161 raise ValidationError(eid, {role_name('config', 'subject'): msg}) |
|
162 elif value is not None: |
|
163 # type check |
|
164 try: |
|
165 value = configuration._validate(value, optdict, optname) |
|
166 except Exception as ex: |
|
167 msg = text_type(ex) # XXX internationalization |
|
168 raise ValidationError(eid, {role_name('config', 'subject'): msg}) |
|
169 processed[optname] = value |
|
170 # cw < 3.10 bw compat |
|
171 try: |
|
172 processed['adapter'] = confdict['adapter'] |
|
173 except KeyError: |
|
174 pass |
|
175 # check for unknown options |
|
176 if confdict and tuple(confdict) != ('adapter',): |
|
177 if fail_if_unknown: |
|
178 msg = _('unknown options %s') % ', '.join(confdict) |
|
179 raise ValidationError(eid, {role_name('config', 'subject'): msg}) |
|
180 else: |
|
181 logger = getLogger('cubicweb.sources') |
|
182 logger.warning('unknown options %s', ', '.join(confdict)) |
|
183 # add options to processed, they may be necessary during migration |
|
184 processed.update(confdict) |
|
185 return processed |
|
186 |
|
187 @classmethod |
|
188 def check_config(cls, source_entity): |
|
189 """check configuration of source entity""" |
|
190 return cls.check_conf_dict(source_entity.eid, source_entity.host_config, |
|
191 _=source_entity._cw._) |
|
192 |
|
193 def update_config(self, source_entity, typedconfig): |
|
194 """update configuration from source entity. `typedconfig` is config |
|
195 properly typed with defaults set |
|
196 """ |
|
197 if source_entity is not None: |
|
198 self._entity_update(source_entity) |
|
199 self.config = typedconfig |
|
200 |
|
201 def _entity_update(self, source_entity): |
|
202 source_entity.complete() |
|
203 if source_entity.url: |
|
204 self.urls = [url.strip() for url in source_entity.url.splitlines() |
|
205 if url.strip()] |
|
206 else: |
|
207 self.urls = [] |
|
208 |
|
209 @staticmethod |
|
210 def decode_extid(extid): |
|
211 if extid is None: |
|
212 return extid |
|
213 return b64decode(extid) |
|
214 |
|
215 # source initialization / finalization ##################################### |
|
216 |
|
217 def set_schema(self, schema): |
|
218 """set the instance'schema""" |
|
219 self.schema = schema |
|
220 |
|
221 def init_creating(self): |
|
222 """method called by the repository once ready to create a new instance""" |
|
223 pass |
|
224 |
|
225 def init(self, activated, source_entity): |
|
226 """method called by the repository once ready to handle request. |
|
227 `activated` is a boolean flag telling if the source is activated or not. |
|
228 """ |
|
229 if activated: |
|
230 self._entity_update(source_entity) |
|
231 |
|
232 PUBLIC_KEYS = ('type', 'uri', 'use-cwuri-as-url') |
|
233 def remove_sensitive_information(self, sourcedef): |
|
234 """remove sensitive information such as login / password from source |
|
235 definition |
|
236 """ |
|
237 for key in list(sourcedef): |
|
238 if not key in self.PUBLIC_KEYS: |
|
239 sourcedef.pop(key) |
|
240 |
|
241 # connections handling ##################################################### |
|
242 |
|
243 def get_connection(self): |
|
244 """open and return a connection to the source""" |
|
245 raise NotImplementedError(self) |
|
246 |
|
247 def close_source_connections(self): |
|
248 for cnxset in self.repo.cnxsets: |
|
249 cnxset.cu = None |
|
250 cnxset.cnx.close() |
|
251 |
|
252 def open_source_connections(self): |
|
253 for cnxset in self.repo.cnxsets: |
|
254 cnxset.cnx = self.get_connection() |
|
255 cnxset.cu = cnxset.cnx.cursor() |
|
256 |
|
257 # cache handling ########################################################### |
|
258 |
|
259 def reset_caches(self): |
|
260 """method called during test to reset potential source caches""" |
|
261 pass |
|
262 |
|
263 def clear_eid_cache(self, eid, etype): |
|
264 """clear potential caches for the given eid""" |
|
265 pass |
|
266 |
|
267 # external source api ###################################################### |
|
268 |
|
269 def support_entity(self, etype, write=False): |
|
270 """return true if the given entity's type is handled by this adapter |
|
271 if write is true, return true only if it's a RW support |
|
272 """ |
|
273 try: |
|
274 wsupport = self.support_entities[etype] |
|
275 except KeyError: |
|
276 return False |
|
277 if write: |
|
278 return wsupport |
|
279 return True |
|
280 |
|
281 def support_relation(self, rtype, write=False): |
|
282 """return true if the given relation's type is handled by this adapter |
|
283 if write is true, return true only if it's a RW support |
|
284 |
|
285 current implementation return true if the relation is defined into |
|
286 `support_relations` or if it is a final relation of a supported entity |
|
287 type |
|
288 """ |
|
289 try: |
|
290 wsupport = self.support_relations[rtype] |
|
291 except KeyError: |
|
292 rschema = self.schema.rschema(rtype) |
|
293 if not rschema.final or rschema.type == 'has_text': |
|
294 return False |
|
295 for etype in rschema.subjects(): |
|
296 try: |
|
297 wsupport = self.support_entities[etype] |
|
298 break |
|
299 except KeyError: |
|
300 continue |
|
301 else: |
|
302 return False |
|
303 if write: |
|
304 return wsupport |
|
305 return True |
|
306 |
|
307 def before_entity_insertion(self, cnx, lid, etype, eid, sourceparams): |
|
308 """called by the repository when an eid has been attributed for an |
|
309 entity stored here but the entity has not been inserted in the system |
|
310 table yet. |
|
311 |
|
312 This method must return the an Entity instance representation of this |
|
313 entity. |
|
314 """ |
|
315 entity = self.repo.vreg['etypes'].etype_class(etype)(cnx) |
|
316 entity.eid = eid |
|
317 entity.cw_edited = EditedEntity(entity) |
|
318 return entity |
|
319 |
|
320 def after_entity_insertion(self, cnx, lid, entity, sourceparams): |
|
321 """called by the repository after an entity stored here has been |
|
322 inserted in the system table. |
|
323 """ |
|
324 pass |
|
325 |
|
326 def _load_mapping(self, cnx, **kwargs): |
|
327 if not 'CWSourceSchemaConfig' in self.schema: |
|
328 self.warning('instance is not mapping ready') |
|
329 return |
|
330 for schemacfg in cnx.execute( |
|
331 'Any CFG,CFGO,S WHERE ' |
|
332 'CFG options CFGO, CFG cw_schema S, ' |
|
333 'CFG cw_for_source X, X eid %(x)s', {'x': self.eid}).entities(): |
|
334 self.add_schema_config(schemacfg, **kwargs) |
|
335 |
|
336 def add_schema_config(self, schemacfg, checkonly=False): |
|
337 """added CWSourceSchemaConfig, modify mapping accordingly""" |
|
338 msg = schemacfg._cw._("this source doesn't use a mapping") |
|
339 raise ValidationError(schemacfg.eid, {None: msg}) |
|
340 |
|
341 def del_schema_config(self, schemacfg, checkonly=False): |
|
342 """deleted CWSourceSchemaConfig, modify mapping accordingly""" |
|
343 msg = schemacfg._cw._("this source doesn't use a mapping") |
|
344 raise ValidationError(schemacfg.eid, {None: msg}) |
|
345 |
|
346 def update_schema_config(self, schemacfg, checkonly=False): |
|
347 """updated CWSourceSchemaConfig, modify mapping accordingly""" |
|
348 self.del_schema_config(schemacfg, checkonly) |
|
349 self.add_schema_config(schemacfg, checkonly) |
|
350 |
|
351 # user authentication api ################################################## |
|
352 |
|
353 def authenticate(self, cnx, login, **kwargs): |
|
354 """if the source support CWUser entity type, it should implement |
|
355 this method which should return CWUser eid for the given login/password |
|
356 if this account is defined in this source and valid login / password is |
|
357 given. Else raise `AuthenticationError` |
|
358 """ |
|
359 raise NotImplementedError(self) |
|
360 |
|
361 # RQL query api ############################################################ |
|
362 |
|
363 def syntax_tree_search(self, cnx, union, |
|
364 args=None, cachekey=None, varmap=None, debug=0): |
|
365 """return result from this source for a rql query (actually from a rql |
|
366 syntax tree and a solution dictionary mapping each used variable to a |
|
367 possible type). If cachekey is given, the query necessary to fetch the |
|
368 results (but not the results themselves) may be cached using this key. |
|
369 """ |
|
370 raise NotImplementedError(self) |
|
371 |
|
372 # write modification api ################################################### |
|
373 # read-only sources don't have to implement methods below |
|
374 |
|
375 def get_extid(self, entity): |
|
376 """return the external id for the given newly inserted entity""" |
|
377 raise NotImplementedError(self) |
|
378 |
|
379 def add_entity(self, cnx, entity): |
|
380 """add a new entity to the source""" |
|
381 raise NotImplementedError(self) |
|
382 |
|
383 def update_entity(self, cnx, entity): |
|
384 """update an entity in the source""" |
|
385 raise NotImplementedError(self) |
|
386 |
|
387 def delete_entities(self, cnx, entities): |
|
388 """delete several entities from the source""" |
|
389 for entity in entities: |
|
390 self.delete_entity(cnx, entity) |
|
391 |
|
392 def delete_entity(self, cnx, entity): |
|
393 """delete an entity from the source""" |
|
394 raise NotImplementedError(self) |
|
395 |
|
396 def add_relation(self, cnx, subject, rtype, object): |
|
397 """add a relation to the source""" |
|
398 raise NotImplementedError(self) |
|
399 |
|
400 def add_relations(self, cnx, rtype, subj_obj_list): |
|
401 """add a relations to the source""" |
|
402 # override in derived classes if you feel you can |
|
403 # optimize |
|
404 for subject, object in subj_obj_list: |
|
405 self.add_relation(cnx, subject, rtype, object) |
|
406 |
|
407 def delete_relation(self, session, subject, rtype, object): |
|
408 """delete a relation from the source""" |
|
409 raise NotImplementedError(self) |
|
410 |
|
411 # system source interface ################################################# |
|
412 |
|
413 def eid_type_source(self, cnx, eid): |
|
414 """return a tuple (type, extid, source) for the entity with id <eid>""" |
|
415 raise NotImplementedError(self) |
|
416 |
|
417 def create_eid(self, cnx): |
|
418 raise NotImplementedError(self) |
|
419 |
|
420 def add_info(self, cnx, entity, source, extid): |
|
421 """add type and source info for an eid into the system table""" |
|
422 raise NotImplementedError(self) |
|
423 |
|
424 def update_info(self, cnx, entity, need_fti_update): |
|
425 """mark entity as being modified, fulltext reindex if needed""" |
|
426 raise NotImplementedError(self) |
|
427 |
|
428 def index_entity(self, cnx, entity): |
|
429 """create an operation to [re]index textual content of the given entity |
|
430 on commit |
|
431 """ |
|
432 raise NotImplementedError(self) |
|
433 |
|
434 def fti_unindex_entities(self, cnx, entities): |
|
435 """remove text content for entities from the full text index |
|
436 """ |
|
437 raise NotImplementedError(self) |
|
438 |
|
439 def fti_index_entities(self, cnx, entities): |
|
440 """add text content of created/modified entities to the full text index |
|
441 """ |
|
442 raise NotImplementedError(self) |
|
443 |
|
444 # sql system source interface ############################################# |
|
445 |
|
446 def sqlexec(self, cnx, sql, args=None): |
|
447 """execute the query and return its result""" |
|
448 raise NotImplementedError(self) |
|
449 |
|
450 def create_index(self, cnx, table, column, unique=False): |
|
451 raise NotImplementedError(self) |
|
452 |
|
453 def drop_index(self, cnx, table, column, unique=False): |
|
454 raise NotImplementedError(self) |
|
455 |
|
456 |
|
457 @deprecated('[3.13] use extid2eid(source, value, etype, cnx, **kwargs)') |
|
458 def extid2eid(self, value, etype, cnx, **kwargs): |
|
459 return self.repo.extid2eid(self, value, etype, cnx, **kwargs) |
|
460 |
|
461 |
|
462 |
|
463 |
|
464 def source_adapter(source_type): |
|
465 try: |
|
466 return SOURCE_TYPES[source_type] |
|
467 except KeyError: |
|
468 raise RuntimeError('Unknown source type %r' % source_type) |
|
469 |
|
470 def get_source(type, source_config, repo, eid): |
|
471 """return a source adapter according to the adapter field in the source's |
|
472 configuration |
|
473 """ |
|
474 return source_adapter(type)(repo, source_config, eid) |