1 # copyright 2003-2014 LOGILAB S.A. (Paris, FRANCE), all rights reserved.

     2 # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr

     3 #

     4 # This file is part of CubicWeb.

     5 #

     6 # CubicWeb is free software: you can redistribute it and/or modify it under the

     7 # terms of the GNU Lesser General Public License as published by the Free

     8 # Software Foundation, either version 2.1 of the License, or (at your option)

     9 # any later version.

    10 #

    11 # CubicWeb is distributed in the hope that it will be useful, but WITHOUT

    12 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS

    13 # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more

    14 # details.

    15 #

    16 # You should have received a copy of the GNU Lesser General Public License along

    17 # with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.

    18 """cubicweb server sources support"""

    19 from __future__ import print_function

    20 

    21 __docformat__ = "restructuredtext en"

    22 

    23 from time import time

    24 from logging import getLogger

    25 from base64 import b64decode

    26 

    27 from six import text_type

    28 

    29 from logilab.common import configuration

    30 from logilab.common.textutils import unormalize

    31 from logilab.common.deprecation import deprecated

    32 

    33 from yams.schema import role_name

    34 

    35 from cubicweb import ValidationError, set_log_methods, server

    36 from cubicweb.server import SOURCE_TYPES

    37 from cubicweb.server.edition import EditedEntity

    38 

    39 

    40 def dbg_st_search(uri, union, varmap, args, cachekey=None, prefix='rql for'):

    41     if server.DEBUG & server.DBG_RQL:

    42         global t

    43         print('  %s %s source: %s' % (prefix, uri, repr(union.as_string())))

    44         t = time()

    45         if varmap:

    46             print('    using varmap', varmap)

    47         if server.DEBUG & server.DBG_MORE:

    48             print('    args', repr(args))

    49             print('    cache key', cachekey)

    50             print('    solutions', ','.join(str(s.solutions)

    51                                             for s in union.children))

    52     # return true so it can be used as assertion (and so be killed by python -O)

    53     return True

    54 

    55 def dbg_results(results):

    56     if server.DEBUG & server.DBG_RQL:

    57         if len(results) > 10:

    58             print('  -->', results[:10], '...', len(results), end=' ')

    59         else:

    60             print('  -->', results, end=' ')

    61         print('time: ', time() - t)

    62     # return true so it can be used as assertion (and so be killed by python -O)

    63     return True

    64 

    65 

    66 class AbstractSource(object):

    67     """an abstract class for sources"""

    68 

    69     # boolean telling if modification hooks should be called when something is

    70     # modified in this source

    71     should_call_hooks = True

    72     # boolean telling if the repository should connect to this source during

    73     # migration

    74     connect_for_migration = True

    75 

    76     # mappings telling which entities and relations are available in the source

    77     # keys are supported entity/relation types and values are boolean indicating

    78     # wether the support is read-only (False) or read-write (True)

    79     support_entities = {}

    80     support_relations = {}

    81     # a global identifier for this source, which has to be set by the source

    82     # instance

    83     uri = None

    84     # a reference to the system information helper

    85     repo = None

    86     # a reference to the instance'schema (may differs from the source'schema)

    87     schema = None

    88 

    89     # force deactivation (configuration error for instance)

    90     disabled = False

    91 

    92     # boolean telling if cwuri of entities from this source is the url that

    93     # should be used as entity's absolute url

    94     use_cwuri_as_url = False

    95 

    96     # source configuration options

    97     options = ()

    98 

    99     # these are overridden by set_log_methods below

   100     # only defining here to prevent pylint from complaining

   101     info = warning = error = critical = exception = debug = lambda msg,*a,**kw: None

   102 

   103     def __init__(self, repo, source_config, eid=None):

   104         self.repo = repo

   105         self.set_schema(repo.schema)

   106         self.support_relations['identity'] = False

   107         self.eid = eid

   108         self.public_config = source_config.copy()

   109         self.public_config['use-cwuri-as-url'] = self.use_cwuri_as_url

   110         self.remove_sensitive_information(self.public_config)

   111         self.uri = source_config.pop('uri')

   112         # unormalize to avoid non-ascii characters in logger's name, this will cause decoding error

   113         # on logging

   114         set_log_methods(self, getLogger('cubicweb.sources.' + unormalize(text_type(self.uri))))

   115         source_config.pop('type')

   116         self.update_config(None, self.check_conf_dict(eid, source_config,

   117                                                       fail_if_unknown=False))

   118 

   119     def __repr__(self):

   120         return '<%s %s source %s @%#x>' % (self.uri, self.__class__.__name__,

   121                                            self.eid, id(self))

   122 

   123     def __lt__(self, other):

   124         """simple comparison function to get predictable source order, with the

   125         system source at last

   126         """

   127         if self.uri == other.uri:

   128             return False

   129         if self.uri == 'system':

   130             return False

   131         if other.uri == 'system':

   132             return True

   133         return self.uri < other.uri

   134 

   135     def __eq__(self, other):

   136         return self.uri == other.uri

   137 

   138     def __ne__(self, other):

   139         return not (self == other)

   140 

   141     def backup(self, backupfile, confirm, format='native'):

   142         """method called to create a backup of source's data"""

   143         pass

   144 

   145     def restore(self, backupfile, confirm, drop, format='native'):

   146         """method called to restore a backup of source's data"""

   147         pass

   148 

   149     @classmethod

   150     def check_conf_dict(cls, eid, confdict, _=text_type, fail_if_unknown=True):

   151         """check configuration of source entity. Return config dict properly

   152         typed with defaults set.

   153         """

   154         processed = {}

   155         for optname, optdict in cls.options:

   156             value = confdict.pop(optname, optdict.get('default'))

   157             if value is configuration.REQUIRED:

   158                 if not fail_if_unknown:

   159                     continue

   160                 msg = _('specifying %s is mandatory' % optname)

   161                 raise ValidationError(eid, {role_name('config', 'subject'): msg})

   162             elif value is not None:

   163                 # type check

   164                 try:

   165                     value = configuration._validate(value, optdict, optname)

   166                 except Exception as ex:

   167                     msg = text_type(ex) # XXX internationalization

   168                     raise ValidationError(eid, {role_name('config', 'subject'): msg})

   169             processed[optname] = value

   170         # cw < 3.10 bw compat

   171         try:

   172             processed['adapter'] = confdict['adapter']

   173         except KeyError:

   174             pass

   175         # check for unknown options

   176         if confdict and tuple(confdict) != ('adapter',):

   177             if fail_if_unknown:

   178                 msg = _('unknown options %s') % ', '.join(confdict)

   179                 raise ValidationError(eid, {role_name('config', 'subject'): msg})

   180             else:

   181                 logger = getLogger('cubicweb.sources')

   182                 logger.warning('unknown options %s', ', '.join(confdict))

   183                 # add options to processed, they may be necessary during migration

   184                 processed.update(confdict)

   185         return processed

   186 

   187     @classmethod

   188     def check_config(cls, source_entity):

   189         """check configuration of source entity"""

   190         return cls.check_conf_dict(source_entity.eid, source_entity.host_config,

   191                                     _=source_entity._cw._)

   192 

   193     def update_config(self, source_entity, typedconfig):

   194         """update configuration from source entity. `typedconfig` is config

   195         properly typed with defaults set

   196         """

   197         if source_entity is not None:

   198             self._entity_update(source_entity)

   199         self.config = typedconfig

   200 

   201     def _entity_update(self, source_entity):

   202         source_entity.complete()

   203         if source_entity.url:

   204             self.urls = [url.strip() for url in source_entity.url.splitlines()

   205                          if url.strip()]

   206         else:

   207             self.urls = []

   208 

   209     @staticmethod

   210     def decode_extid(extid):

   211         if extid is None:

   212             return extid

   213         return b64decode(extid)

   214 

   215     # source initialization / finalization #####################################

   216 

   217     def set_schema(self, schema):

   218         """set the instance'schema"""

   219         self.schema = schema

   220 

   221     def init_creating(self):

   222         """method called by the repository once ready to create a new instance"""

   223         pass

   224 

   225     def init(self, activated, source_entity):

   226         """method called by the repository once ready to handle request.

   227         `activated` is a boolean flag telling if the source is activated or not.

   228         """

   229         if activated:

   230             self._entity_update(source_entity)

   231 

   232     PUBLIC_KEYS = ('type', 'uri', 'use-cwuri-as-url')

   233     def remove_sensitive_information(self, sourcedef):

   234         """remove sensitive information such as login / password from source

   235         definition

   236         """

   237         for key in list(sourcedef):

   238             if not key in self.PUBLIC_KEYS:

   239                 sourcedef.pop(key)

   240 

   241     # connections handling #####################################################

   242 

   243     def get_connection(self):

   244         """open and return a connection to the source"""

   245         raise NotImplementedError(self)

   246 

   247     def close_source_connections(self):

   248         for cnxset in self.repo.cnxsets:

   249             cnxset.cu = None

   250             cnxset.cnx.close()

   251 

   252     def open_source_connections(self):

   253         for cnxset in self.repo.cnxsets:

   254             cnxset.cnx = self.get_connection()

   255             cnxset.cu = cnxset.cnx.cursor()

   256 

   257     # cache handling ###########################################################

   258 

   259     def reset_caches(self):

   260         """method called during test to reset potential source caches"""

   261         pass

   262 

   263     def clear_eid_cache(self, eid, etype):

   264         """clear potential caches for the given eid"""

   265         pass

   266 

   267     # external source api ######################################################

   268 

   269     def support_entity(self, etype, write=False):

   270         """return true if the given entity's type is handled by this adapter

   271         if write is true, return true only if it's a RW support

   272         """

   273         try:

   274             wsupport = self.support_entities[etype]

   275         except KeyError:

   276             return False

   277         if write:

   278             return wsupport

   279         return True

   280 

   281     def support_relation(self, rtype, write=False):

   282         """return true if the given relation's type is handled by this adapter

   283         if write is true, return true only if it's a RW support

   284 

   285         current implementation return true if the relation is defined into

   286         `support_relations` or if it is a final relation of a supported entity

   287         type

   288         """

   289         try:

   290             wsupport = self.support_relations[rtype]

   291         except KeyError:

   292             rschema = self.schema.rschema(rtype)

   293             if not rschema.final or rschema.type == 'has_text':

   294                 return False

   295             for etype in rschema.subjects():

   296                 try:

   297                     wsupport = self.support_entities[etype]

   298                     break

   299                 except KeyError:

   300                     continue

   301             else:

   302                 return False

   303         if write:

   304             return wsupport

   305         return True

   306 

   307     def before_entity_insertion(self, cnx, lid, etype, eid, sourceparams):

   308         """called by the repository when an eid has been attributed for an

   309         entity stored here but the entity has not been inserted in the system

   310         table yet.

   311 

   312         This method must return the an Entity instance representation of this

   313         entity.

   314         """

   315         entity = self.repo.vreg['etypes'].etype_class(etype)(cnx)

   316         entity.eid = eid

   317         entity.cw_edited = EditedEntity(entity)

   318         return entity

   319 

   320     def after_entity_insertion(self, cnx, lid, entity, sourceparams):

   321         """called by the repository after an entity stored here has been

   322         inserted in the system table.

   323         """

   324         pass

   325 

   326     def _load_mapping(self, cnx, **kwargs):

   327         if not 'CWSourceSchemaConfig' in self.schema:

   328             self.warning('instance is not mapping ready')

   329             return

   330         for schemacfg in cnx.execute(

   331             'Any CFG,CFGO,S WHERE '

   332             'CFG options CFGO, CFG cw_schema S, '

   333             'CFG cw_for_source X, X eid %(x)s', {'x': self.eid}).entities():

   334             self.add_schema_config(schemacfg, **kwargs)

   335 

   336     def add_schema_config(self, schemacfg, checkonly=False):

   337         """added CWSourceSchemaConfig, modify mapping accordingly"""

   338         msg = schemacfg._cw._("this source doesn't use a mapping")

   339         raise ValidationError(schemacfg.eid, {None: msg})

   340 

   341     def del_schema_config(self, schemacfg, checkonly=False):

   342         """deleted CWSourceSchemaConfig, modify mapping accordingly"""

   343         msg = schemacfg._cw._("this source doesn't use a mapping")

   344         raise ValidationError(schemacfg.eid, {None: msg})

   345 

   346     def update_schema_config(self, schemacfg, checkonly=False):

   347         """updated CWSourceSchemaConfig, modify mapping accordingly"""

   348         self.del_schema_config(schemacfg, checkonly)

   349         self.add_schema_config(schemacfg, checkonly)

   350 

   351     # user authentication api ##################################################

   352 

   353     def authenticate(self, cnx, login, **kwargs):

   354         """if the source support CWUser entity type, it should implement

   355         this method which should return CWUser eid for the given login/password

   356         if this account is defined in this source and valid login / password is

   357         given. Else raise `AuthenticationError`

   358         """

   359         raise NotImplementedError(self)

   360 

   361     # RQL query api ############################################################

   362 

   363     def syntax_tree_search(self, cnx, union,

   364                            args=None, cachekey=None, varmap=None, debug=0):

   365         """return result from this source for a rql query (actually from a rql

   366         syntax tree and a solution dictionary mapping each used variable to a

   367         possible type). If cachekey is given, the query necessary to fetch the

   368         results (but not the results themselves) may be cached using this key.

   369         """

   370         raise NotImplementedError(self)

   371 

   372     # write modification api ###################################################

   373     # read-only sources don't have to implement methods below

   374 

   375     def get_extid(self, entity):

   376         """return the external id for the given newly inserted entity"""

   377         raise NotImplementedError(self)

   378 

   379     def add_entity(self, cnx, entity):

   380         """add a new entity to the source"""

   381         raise NotImplementedError(self)

   382 

   383     def update_entity(self, cnx, entity):

   384         """update an entity in the source"""

   385         raise NotImplementedError(self)

   386 

   387     def delete_entities(self, cnx, entities):

   388         """delete several entities from the source"""

   389         for entity in entities:

   390             self.delete_entity(cnx, entity)

   391 

   392     def delete_entity(self, cnx, entity):

   393         """delete an entity from the source"""

   394         raise NotImplementedError(self)

   395 

   396     def add_relation(self, cnx, subject, rtype, object):

   397         """add a relation to the source"""

   398         raise NotImplementedError(self)

   399 

   400     def add_relations(self, cnx,  rtype, subj_obj_list):

   401         """add a relations to the source"""

   402         # override in derived classes if you feel you can

   403         # optimize

   404         for subject, object in subj_obj_list:

   405             self.add_relation(cnx, subject, rtype, object)

   406 

   407     def delete_relation(self, session, subject, rtype, object):

   408         """delete a relation from the source"""

   409         raise NotImplementedError(self)

   410 

   411     # system source interface #################################################

   412 

   413     def eid_type_source(self, cnx, eid):

   414         """return a tuple (type, extid, source) for the entity with id <eid>"""

   415         raise NotImplementedError(self)

   416 

   417     def create_eid(self, cnx):

   418         raise NotImplementedError(self)

   419 

   420     def add_info(self, cnx, entity, source, extid):

   421         """add type and source info for an eid into the system table"""

   422         raise NotImplementedError(self)

   423 

   424     def update_info(self, cnx, entity, need_fti_update):

   425         """mark entity as being modified, fulltext reindex if needed"""

   426         raise NotImplementedError(self)

   427 

   428     def index_entity(self, cnx, entity):

   429         """create an operation to [re]index textual content of the given entity

   430         on commit

   431         """

   432         raise NotImplementedError(self)

   433 

   434     def fti_unindex_entities(self, cnx, entities):

   435         """remove text content for entities from the full text index

   436         """

   437         raise NotImplementedError(self)

   438 

   439     def fti_index_entities(self, cnx, entities):

   440         """add text content of created/modified entities to the full text index

   441         """

   442         raise NotImplementedError(self)

   443 

   444     # sql system source interface #############################################

   445 

   446     def sqlexec(self, cnx, sql, args=None):

   447         """execute the query and return its result"""

   448         raise NotImplementedError(self)

   449 

   450     def create_index(self, cnx, table, column, unique=False):

   451         raise NotImplementedError(self)

   452 

   453     def drop_index(self, cnx, table, column, unique=False):

   454         raise NotImplementedError(self)

   455 

   456 

   457     @deprecated('[3.13] use extid2eid(source, value, etype, cnx, **kwargs)')

   458     def extid2eid(self, value, etype, cnx, **kwargs):

   459         return self.repo.extid2eid(self, value, etype, cnx, **kwargs)

   460 

   461 

   462 

   463 

   464 def source_adapter(source_type):

   465     try:

   466         return SOURCE_TYPES[source_type]

   467     except KeyError:

   468         raise RuntimeError('Unknown source type %r' % source_type)

   469 

   470 def get_source(type, source_config, repo, eid):

   471     """return a source adapter according to the adapter field in the source's

   472     configuration

   473     """

   474     return source_adapter(type)(repo, source_config, eid)