server/sources/__init__.py
author Julien Cristau <julien.cristau@logilab.fr>
Wed, 06 Jan 2016 11:47:08 +0100
changeset 11080 245f440b814a
parent 10845 75c60e58ce6b
permissions -rw-r--r--
[devtools] fix get_bounds Say an attribute has two IntervalBoundConstraints, one with minvalue=0 and the other with maxvalue=10 (not literally, or that could be a single constraint, but e.g. with references to other attributes). We don't want to override the minvalue of the former with None just because the latter happens last in the constraints list. This doesn't fix the case where two constraints both have a min or a max value. Meh.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
10086
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9822
diff changeset
     1
# copyright 2003-2014 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
5421
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5124
diff changeset
     2
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5124
diff changeset
     3
#
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5124
diff changeset
     4
# This file is part of CubicWeb.
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5124
diff changeset
     5
#
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5124
diff changeset
     6
# CubicWeb is free software: you can redistribute it and/or modify it under the
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5124
diff changeset
     7
# terms of the GNU Lesser General Public License as published by the Free
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5124
diff changeset
     8
# Software Foundation, either version 2.1 of the License, or (at your option)
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5124
diff changeset
     9
# any later version.
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5124
diff changeset
    10
#
5424
8ecbcbff9777 replace logilab-common by CubicWeb in disclaimer
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5421
diff changeset
    11
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
5421
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5124
diff changeset
    12
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5124
diff changeset
    13
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5124
diff changeset
    14
# details.
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5124
diff changeset
    15
#
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5124
diff changeset
    16
# You should have received a copy of the GNU Lesser General Public License along
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5124
diff changeset
    17
# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
5824
de9b7e88660e cleanup
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5642
diff changeset
    18
"""cubicweb server sources support"""
10589
7c23b7de2b8d [py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents: 10580
diff changeset
    19
from __future__ import print_function
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    20
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    21
__docformat__ = "restructuredtext en"
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    22
9180
13ed6de41774 [querier] Add timings to debug prints (DBG_RQL)
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 8892
diff changeset
    23
from time import time
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    24
from logging import getLogger
10580
5fbdbbe51867 [server/sources] add decode_extid method
Julien Cristau <julien.cristau@logilab.fr>
parents: 10086
diff changeset
    25
from base64 import b64decode
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    26
10622
3cc6154b94a3 [py3k] unicode → six.text_type
Rémi Cardona <remi.cardona@logilab.fr>
parents: 10589
diff changeset
    27
from six import text_type
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    28
6945
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
    29
from logilab.common import configuration
10832
2350424585f8 [server] unormalize source logger's name to avoid encoding errors
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10651
diff changeset
    30
from logilab.common.textutils import unormalize
7697
ef50074a0314 [repo api] deprecates source.extid2eid, only the system source should implement it,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7552
diff changeset
    31
from logilab.common.deprecation import deprecated
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    32
6945
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
    33
from yams.schema import role_name
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
    34
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
    35
from cubicweb import ValidationError, set_log_methods, server
10086
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9822
diff changeset
    36
from cubicweb.server import SOURCE_TYPES
7118
e094b3d4eb95 [server] move EditedEntity class to its own module, to avoid cyclic dependency when needed from e.g. session.py
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7040
diff changeset
    37
from cubicweb.server.edition import EditedEntity
1251
af40e615dc89 introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents: 1250
diff changeset
    38
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    39
2625
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2596
diff changeset
    40
def dbg_st_search(uri, union, varmap, args, cachekey=None, prefix='rql for'):
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2596
diff changeset
    41
    if server.DEBUG & server.DBG_RQL:
9180
13ed6de41774 [querier] Add timings to debug prints (DBG_RQL)
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 8892
diff changeset
    42
        global t
10589
7c23b7de2b8d [py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents: 10580
diff changeset
    43
        print('  %s %s source: %s' % (prefix, uri, repr(union.as_string())))
9180
13ed6de41774 [querier] Add timings to debug prints (DBG_RQL)
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 8892
diff changeset
    44
        t = time()
2625
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2596
diff changeset
    45
        if varmap:
10589
7c23b7de2b8d [py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents: 10580
diff changeset
    46
            print('    using varmap', varmap)
2625
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2596
diff changeset
    47
        if server.DEBUG & server.DBG_MORE:
10589
7c23b7de2b8d [py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents: 10580
diff changeset
    48
            print('    args', repr(args))
7c23b7de2b8d [py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents: 10580
diff changeset
    49
            print('    cache key', cachekey)
7c23b7de2b8d [py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents: 10580
diff changeset
    50
            print('    solutions', ','.join(str(s.solutions)
7c23b7de2b8d [py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents: 10580
diff changeset
    51
                                            for s in union.children))
2625
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2596
diff changeset
    52
    # return true so it can be used as assertion (and so be killed by python -O)
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2596
diff changeset
    53
    return True
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2596
diff changeset
    54
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2596
diff changeset
    55
def dbg_results(results):
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2596
diff changeset
    56
    if server.DEBUG & server.DBG_RQL:
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2596
diff changeset
    57
        if len(results) > 10:
10589
7c23b7de2b8d [py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents: 10580
diff changeset
    58
            print('  -->', results[:10], '...', len(results), end=' ')
2625
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2596
diff changeset
    59
        else:
10589
7c23b7de2b8d [py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents: 10580
diff changeset
    60
            print('  -->', results, end=' ')
7c23b7de2b8d [py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents: 10580
diff changeset
    61
        print('time: ', time() - t)
2625
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2596
diff changeset
    62
    # return true so it can be used as assertion (and so be killed by python -O)
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2596
diff changeset
    63
    return True
2493
9806571ea790 major refactoring of database dump/restore:
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2476
diff changeset
    64
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    65
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    66
class AbstractSource(object):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    67
    """an abstract class for sources"""
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    68
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    69
    # boolean telling if modification hooks should be called when something is
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    70
    # modified in this source
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    71
    should_call_hooks = True
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    72
    # boolean telling if the repository should connect to this source during
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    73
    # migration
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    74
    connect_for_migration = True
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
    75
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    76
    # mappings telling which entities and relations are available in the source
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    77
    # keys are supported entity/relation types and values are boolean indicating
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    78
    # wether the support is read-only (False) or read-write (True)
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    79
    support_entities = {}
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    80
    support_relations = {}
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    81
    # a global identifier for this source, which has to be set by the source
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    82
    # instance
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    83
    uri = None
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    84
    # a reference to the system information helper
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    85
    repo = None
2476
1294a6bdf3bf application -> instance where it makes sense
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1977
diff changeset
    86
    # a reference to the instance'schema (may differs from the source'schema)
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    87
    schema = None
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
    88
6427
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6162
diff changeset
    89
    # force deactivation (configuration error for instance)
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6162
diff changeset
    90
    disabled = False
3040
e4dc22040f5f [multi-sources] move [dont_]cross_relations to AbstractSource
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2759
diff changeset
    91
7552
82dde8276a5b [datafeed, entities] url for entities from a datafeed source should be on their origin site. Closes #1769391
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7514
diff changeset
    92
    # boolean telling if cwuri of entities from this source is the url that
82dde8276a5b [datafeed, entities] url for entities from a datafeed source should be on their origin site. Closes #1769391
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7514
diff changeset
    93
    # should be used as entity's absolute url
82dde8276a5b [datafeed, entities] url for entities from a datafeed source should be on their origin site. Closes #1769391
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7514
diff changeset
    94
    use_cwuri_as_url = False
82dde8276a5b [datafeed, entities] url for entities from a datafeed source should be on their origin site. Closes #1769391
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7514
diff changeset
    95
6945
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
    96
    # source configuration options
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
    97
    options = ()
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
    98
7879
9aae456abab5 [pylint] fix pylint detected errors and tweak it so that pylint -E will be much less verbose next time (+ update some copyrights on the way)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7815
diff changeset
    99
    # these are overridden by set_log_methods below
9aae456abab5 [pylint] fix pylint detected errors and tweak it so that pylint -E will be much less verbose next time (+ update some copyrights on the way)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7815
diff changeset
   100
    # only defining here to prevent pylint from complaining
9aae456abab5 [pylint] fix pylint detected errors and tweak it so that pylint -E will be much less verbose next time (+ update some copyrights on the way)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7815
diff changeset
   101
    info = warning = error = critical = exception = debug = lambda msg,*a,**kw: None
9aae456abab5 [pylint] fix pylint detected errors and tweak it so that pylint -E will be much less verbose next time (+ update some copyrights on the way)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7815
diff changeset
   102
6945
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   103
    def __init__(self, repo, source_config, eid=None):
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   104
        self.repo = repo
6427
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6162
diff changeset
   105
        self.set_schema(repo.schema)
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   106
        self.support_relations['identity'] = False
6945
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   107
        self.eid = eid
6722
3341521d857b [repo source] rename attribute so it's much easier to grasp its role
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6582
diff changeset
   108
        self.public_config = source_config.copy()
9822
4a118bfd6ab4 [datafeed] Allow to override use_cwuri_as_url in configuration of a datafeed source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9662
diff changeset
   109
        self.public_config['use-cwuri-as-url'] = self.use_cwuri_as_url
6722
3341521d857b [repo source] rename attribute so it's much easier to grasp its role
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6582
diff changeset
   110
        self.remove_sensitive_information(self.public_config)
6945
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   111
        self.uri = source_config.pop('uri')
10832
2350424585f8 [server] unormalize source logger's name to avoid encoding errors
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10651
diff changeset
   112
        # unormalize to avoid non-ascii characters in logger's name, this will cause decoding error
2350424585f8 [server] unormalize source logger's name to avoid encoding errors
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10651
diff changeset
   113
        # on logging
10845
75c60e58ce6b [py3k] Fix unicode added in commit 2350424585f8
Rémi Cardona <remi.cardona@logilab.fr>
parents: 10832
diff changeset
   114
        set_log_methods(self, getLogger('cubicweb.sources.' + unormalize(text_type(self.uri))))
6945
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   115
        source_config.pop('type')
8674
001c1592060a [repo sources] move handling of source's url into abstract source as this becomes shared by most sources
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8544
diff changeset
   116
        self.update_config(None, self.check_conf_dict(eid, source_config,
001c1592060a [repo sources] move handling of source's url into abstract source as this becomes shared by most sources
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8544
diff changeset
   117
                                                      fail_if_unknown=False))
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   118
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   119
    def __repr__(self):
8228
fe53c7e1625f [sources] include source's class in its repr
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8134
diff changeset
   120
        return '<%s %s source %s @%#x>' % (self.uri, self.__class__.__name__,
fe53c7e1625f [sources] include source's class in its repr
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8134
diff changeset
   121
                                           self.eid, id(self))
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   122
8892
80783605d270 [toward-py3k] rewrite __cmp__ (closes #2715115)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents: 8696
diff changeset
   123
    def __lt__(self, other):
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   124
        """simple comparison function to get predictable source order, with the
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   125
        system source at last
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   126
        """
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   127
        if self.uri == other.uri:
8892
80783605d270 [toward-py3k] rewrite __cmp__ (closes #2715115)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents: 8696
diff changeset
   128
            return False
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   129
        if self.uri == 'system':
8892
80783605d270 [toward-py3k] rewrite __cmp__ (closes #2715115)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents: 8696
diff changeset
   130
            return False
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   131
        if other.uri == 'system':
8892
80783605d270 [toward-py3k] rewrite __cmp__ (closes #2715115)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents: 8696
diff changeset
   132
            return True
80783605d270 [toward-py3k] rewrite __cmp__ (closes #2715115)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents: 8696
diff changeset
   133
        return self.uri < other.uri
80783605d270 [toward-py3k] rewrite __cmp__ (closes #2715115)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents: 8696
diff changeset
   134
80783605d270 [toward-py3k] rewrite __cmp__ (closes #2715115)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents: 8696
diff changeset
   135
    def __eq__(self, other):
80783605d270 [toward-py3k] rewrite __cmp__ (closes #2715115)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents: 8696
diff changeset
   136
        return self.uri == other.uri
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   137
10576
1ee631aedf2f [server/sources] AbstractSource.__ne__
Julien Cristau <julien.cristau@logilab.fr>
parents: 10575
diff changeset
   138
    def __ne__(self, other):
1ee631aedf2f [server/sources] AbstractSource.__ne__
Julien Cristau <julien.cristau@logilab.fr>
parents: 10575
diff changeset
   139
        return not (self == other)
1ee631aedf2f [server/sources] AbstractSource.__ne__
Julien Cristau <julien.cristau@logilab.fr>
parents: 10575
diff changeset
   140
7342
d1c8b5b3531c adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7237
diff changeset
   141
    def backup(self, backupfile, confirm, format='native'):
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   142
        """method called to create a backup of source's data"""
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   143
        pass
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   144
7342
d1c8b5b3531c adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7237
diff changeset
   145
    def restore(self, backupfile, confirm, drop, format='native'):
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   146
        """method called to restore a backup of source's data"""
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   147
        pass
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   148
6945
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   149
    @classmethod
10622
3cc6154b94a3 [py3k] unicode → six.text_type
Rémi Cardona <remi.cardona@logilab.fr>
parents: 10589
diff changeset
   150
    def check_conf_dict(cls, eid, confdict, _=text_type, fail_if_unknown=True):
6945
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   151
        """check configuration of source entity. Return config dict properly
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   152
        typed with defaults set.
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   153
        """
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   154
        processed = {}
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   155
        for optname, optdict in cls.options:
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   156
            value = confdict.pop(optname, optdict.get('default'))
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   157
            if value is configuration.REQUIRED:
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   158
                if not fail_if_unknown:
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   159
                    continue
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   160
                msg = _('specifying %s is mandatory' % optname)
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   161
                raise ValidationError(eid, {role_name('config', 'subject'): msg})
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   162
            elif value is not None:
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   163
                # type check
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   164
                try:
9292
4b3e657d17ab [source configuration] silent lgc 0.60 deprecation warning
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9180
diff changeset
   165
                    value = configuration._validate(value, optdict, optname)
8695
358d8bed9626 [toward-py3k] rewrite to "except AnException as exc:" (part of #2711624)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents: 8674
diff changeset
   166
                except Exception as ex:
10622
3cc6154b94a3 [py3k] unicode → six.text_type
Rémi Cardona <remi.cardona@logilab.fr>
parents: 10589
diff changeset
   167
                    msg = text_type(ex) # XXX internationalization
6945
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   168
                    raise ValidationError(eid, {role_name('config', 'subject'): msg})
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   169
            processed[optname] = value
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   170
        # cw < 3.10 bw compat
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   171
        try:
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   172
            processed['adapter'] = confdict['adapter']
7815
2a164a9cf81c [exceptions] stop catching any exception in various places (closes #1942716)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7779
diff changeset
   173
        except KeyError:
6945
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   174
            pass
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   175
        # check for unknown options
8696
0bb18407c053 [toward py3k] rewrite dict.keys() and dict.values() (part of #2711624)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents: 8695
diff changeset
   176
        if confdict and tuple(confdict) != ('adapter',):
6945
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   177
            if fail_if_unknown:
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   178
                msg = _('unknown options %s') % ', '.join(confdict)
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   179
                raise ValidationError(eid, {role_name('config', 'subject'): msg})
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   180
            else:
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   181
                logger = getLogger('cubicweb.sources')
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   182
                logger.warning('unknown options %s', ', '.join(confdict))
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   183
                # add options to processed, they may be necessary during migration
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   184
                processed.update(confdict)
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   185
        return processed
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   186
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   187
    @classmethod
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   188
    def check_config(cls, source_entity):
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   189
        """check configuration of source entity"""
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   190
        return cls.check_conf_dict(source_entity.eid, source_entity.host_config,
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   191
                                    _=source_entity._cw._)
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   192
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   193
    def update_config(self, source_entity, typedconfig):
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   194
        """update configuration from source entity. `typedconfig` is config
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   195
        properly typed with defaults set
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   196
        """
8674
001c1592060a [repo sources] move handling of source's url into abstract source as this becomes shared by most sources
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8544
diff changeset
   197
        if source_entity is not None:
001c1592060a [repo sources] move handling of source's url into abstract source as this becomes shared by most sources
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8544
diff changeset
   198
            self._entity_update(source_entity)
001c1592060a [repo sources] move handling of source's url into abstract source as this becomes shared by most sources
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8544
diff changeset
   199
        self.config = typedconfig
001c1592060a [repo sources] move handling of source's url into abstract source as this becomes shared by most sources
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8544
diff changeset
   200
001c1592060a [repo sources] move handling of source's url into abstract source as this becomes shared by most sources
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8544
diff changeset
   201
    def _entity_update(self, source_entity):
001c1592060a [repo sources] move handling of source's url into abstract source as this becomes shared by most sources
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8544
diff changeset
   202
        source_entity.complete()
001c1592060a [repo sources] move handling of source's url into abstract source as this becomes shared by most sources
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8544
diff changeset
   203
        if source_entity.url:
001c1592060a [repo sources] move handling of source's url into abstract source as this becomes shared by most sources
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8544
diff changeset
   204
            self.urls = [url.strip() for url in source_entity.url.splitlines()
001c1592060a [repo sources] move handling of source's url into abstract source as this becomes shared by most sources
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8544
diff changeset
   205
                         if url.strip()]
001c1592060a [repo sources] move handling of source's url into abstract source as this becomes shared by most sources
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8544
diff changeset
   206
        else:
001c1592060a [repo sources] move handling of source's url into abstract source as this becomes shared by most sources
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8544
diff changeset
   207
            self.urls = []
6945
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   208
10580
5fbdbbe51867 [server/sources] add decode_extid method
Julien Cristau <julien.cristau@logilab.fr>
parents: 10086
diff changeset
   209
    @staticmethod
5fbdbbe51867 [server/sources] add decode_extid method
Julien Cristau <julien.cristau@logilab.fr>
parents: 10086
diff changeset
   210
    def decode_extid(extid):
5fbdbbe51867 [server/sources] add decode_extid method
Julien Cristau <julien.cristau@logilab.fr>
parents: 10086
diff changeset
   211
        if extid is None:
5fbdbbe51867 [server/sources] add decode_extid method
Julien Cristau <julien.cristau@logilab.fr>
parents: 10086
diff changeset
   212
            return extid
5fbdbbe51867 [server/sources] add decode_extid method
Julien Cristau <julien.cristau@logilab.fr>
parents: 10086
diff changeset
   213
        return b64decode(extid)
5fbdbbe51867 [server/sources] add decode_extid method
Julien Cristau <julien.cristau@logilab.fr>
parents: 10086
diff changeset
   214
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   215
    # source initialization / finalization #####################################
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   216
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   217
    def set_schema(self, schema):
2476
1294a6bdf3bf application -> instance where it makes sense
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1977
diff changeset
   218
        """set the instance'schema"""
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   219
        self.schema = schema
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   220
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   221
    def init_creating(self):
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   222
        """method called by the repository once ready to create a new instance"""
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   223
        pass
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   224
6957
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
   225
    def init(self, activated, source_entity):
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   226
        """method called by the repository once ready to handle request.
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   227
        `activated` is a boolean flag telling if the source is activated or not.
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   228
        """
8674
001c1592060a [repo sources] move handling of source's url into abstract source as this becomes shared by most sources
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8544
diff changeset
   229
        if activated:
001c1592060a [repo sources] move handling of source's url into abstract source as this becomes shared by most sources
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8544
diff changeset
   230
            self._entity_update(source_entity)
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   231
7552
82dde8276a5b [datafeed, entities] url for entities from a datafeed source should be on their origin site. Closes #1769391
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7514
diff changeset
   232
    PUBLIC_KEYS = ('type', 'uri', 'use-cwuri-as-url')
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   233
    def remove_sensitive_information(self, sourcedef):
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   234
        """remove sensitive information such as login / password from source
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   235
        definition
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   236
        """
8696
0bb18407c053 [toward py3k] rewrite dict.keys() and dict.values() (part of #2711624)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents: 8695
diff changeset
   237
        for key in list(sourcedef):
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   238
            if not key in self.PUBLIC_KEYS:
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   239
                sourcedef.pop(key)
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   240
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   241
    # connections handling #####################################################
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   242
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   243
    def get_connection(self):
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   244
        """open and return a connection to the source"""
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   245
        raise NotImplementedError(self)
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   246
7398
26695dd703d8 [repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7342
diff changeset
   247
    def close_source_connections(self):
26695dd703d8 [repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7342
diff changeset
   248
        for cnxset in self.repo.cnxsets:
9463
d62e13eba033 [multi-sources-removal] Simplify ConnectionsSet internal structures and public methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9452
diff changeset
   249
            cnxset.cu = None
d62e13eba033 [multi-sources-removal] Simplify ConnectionsSet internal structures and public methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9452
diff changeset
   250
            cnxset.cnx.close()
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   251
7398
26695dd703d8 [repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7342
diff changeset
   252
    def open_source_connections(self):
26695dd703d8 [repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7342
diff changeset
   253
        for cnxset in self.repo.cnxsets:
9463
d62e13eba033 [multi-sources-removal] Simplify ConnectionsSet internal structures and public methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9452
diff changeset
   254
            cnxset.cnx = self.get_connection()
d62e13eba033 [multi-sources-removal] Simplify ConnectionsSet internal structures and public methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9452
diff changeset
   255
            cnxset.cu = cnxset.cnx.cursor()
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   256
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   257
    # cache handling ###########################################################
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   258
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   259
    def reset_caches(self):
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   260
        """method called during test to reset potential source caches"""
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   261
        pass
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   262
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   263
    def clear_eid_cache(self, eid, etype):
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   264
        """clear potential caches for the given eid"""
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   265
        pass
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   266
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   267
    # external source api ######################################################
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   268
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   269
    def support_entity(self, etype, write=False):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   270
        """return true if the given entity's type is handled by this adapter
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   271
        if write is true, return true only if it's a RW support
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   272
        """
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   273
        try:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   274
            wsupport = self.support_entities[etype]
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   275
        except KeyError:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   276
            return False
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   277
        if write:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   278
            return wsupport
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   279
        return True
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   280
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   281
    def support_relation(self, rtype, write=False):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   282
        """return true if the given relation's type is handled by this adapter
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   283
        if write is true, return true only if it's a RW support
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   284
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   285
        current implementation return true if the relation is defined into
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   286
        `support_relations` or if it is a final relation of a supported entity
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   287
        type
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   288
        """
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   289
        try:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   290
            wsupport = self.support_relations[rtype]
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   291
        except KeyError:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   292
            rschema = self.schema.rschema(rtype)
6057
cb5c8852cbda [ms] cleanups
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5849
diff changeset
   293
            if not rschema.final or rschema.type == 'has_text':
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   294
                return False
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   295
            for etype in rschema.subjects():
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   296
                try:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   297
                    wsupport = self.support_entities[etype]
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   298
                    break
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   299
                except KeyError:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   300
                    continue
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   301
            else:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   302
                return False
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   303
        if write:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   304
            return wsupport
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   305
        return True
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   306
9620
7b7d5a5d6365 [server/sources] {before,after}_entity_insertion wants a cnx not a session
Julien Cristau <julien.cristau@logilab.fr>
parents: 9543
diff changeset
   307
    def before_entity_insertion(self, cnx, lid, etype, eid, sourceparams):
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   308
        """called by the repository when an eid has been attributed for an
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   309
        entity stored here but the entity has not been inserted in the system
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   310
        table yet.
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   311
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   312
        This method must return the an Entity instance representation of this
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   313
        entity.
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   314
        """
9620
7b7d5a5d6365 [server/sources] {before,after}_entity_insertion wants a cnx not a session
Julien Cristau <julien.cristau@logilab.fr>
parents: 9543
diff changeset
   315
        entity = self.repo.vreg['etypes'].etype_class(etype)(cnx)
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   316
        entity.eid = eid
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   317
        entity.cw_edited = EditedEntity(entity)
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   318
        return entity
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   319
9620
7b7d5a5d6365 [server/sources] {before,after}_entity_insertion wants a cnx not a session
Julien Cristau <julien.cristau@logilab.fr>
parents: 9543
diff changeset
   320
    def after_entity_insertion(self, cnx, lid, entity, sourceparams):
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   321
        """called by the repository after an entity stored here has been
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   322
        inserted in the system table.
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   323
        """
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   324
        pass
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   325
10086
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9822
diff changeset
   326
    def _load_mapping(self, cnx, **kwargs):
6957
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
   327
        if not 'CWSourceSchemaConfig' in self.schema:
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
   328
            self.warning('instance is not mapping ready')
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
   329
            return
10086
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9822
diff changeset
   330
        for schemacfg in cnx.execute(
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9822
diff changeset
   331
            'Any CFG,CFGO,S WHERE '
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9822
diff changeset
   332
            'CFG options CFGO, CFG cw_schema S, '
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9822
diff changeset
   333
            'CFG cw_for_source X, X eid %(x)s', {'x': self.eid}).entities():
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9822
diff changeset
   334
            self.add_schema_config(schemacfg, **kwargs)
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   335
6944
0cf10429ad39 [sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6943
diff changeset
   336
    def add_schema_config(self, schemacfg, checkonly=False):
0cf10429ad39 [sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6943
diff changeset
   337
        """added CWSourceSchemaConfig, modify mapping accordingly"""
0cf10429ad39 [sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6943
diff changeset
   338
        msg = schemacfg._cw._("this source doesn't use a mapping")
0cf10429ad39 [sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6943
diff changeset
   339
        raise ValidationError(schemacfg.eid, {None: msg})
0cf10429ad39 [sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6943
diff changeset
   340
0cf10429ad39 [sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6943
diff changeset
   341
    def del_schema_config(self, schemacfg, checkonly=False):
0cf10429ad39 [sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6943
diff changeset
   342
        """deleted CWSourceSchemaConfig, modify mapping accordingly"""
0cf10429ad39 [sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6943
diff changeset
   343
        msg = schemacfg._cw._("this source doesn't use a mapping")
0cf10429ad39 [sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6943
diff changeset
   344
        raise ValidationError(schemacfg.eid, {None: msg})
0cf10429ad39 [sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6943
diff changeset
   345
0cf10429ad39 [sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6943
diff changeset
   346
    def update_schema_config(self, schemacfg, checkonly=False):
0cf10429ad39 [sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6943
diff changeset
   347
        """updated CWSourceSchemaConfig, modify mapping accordingly"""
0cf10429ad39 [sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6943
diff changeset
   348
        self.del_schema_config(schemacfg, checkonly)
0cf10429ad39 [sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6943
diff changeset
   349
        self.add_schema_config(schemacfg, checkonly)
0cf10429ad39 [sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6943
diff changeset
   350
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   351
    # user authentication api ##################################################
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   352
9512
88dc96fc9fc1 [server] use a connection instead of a session for user authentication
Julien Cristau <julien.cristau@logilab.fr>
parents: 9469
diff changeset
   353
    def authenticate(self, cnx, login, **kwargs):
6152
6824f8b61098 use is_instance in a number of places (esp. documentation) rather than the deprecated implements
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 6057
diff changeset
   354
        """if the source support CWUser entity type, it should implement
1398
5fe84a5f7035 rename internal entity types to have CW prefix instead of E
sylvain.thenault@logilab.fr
parents: 1263
diff changeset
   355
        this method which should return CWUser eid for the given login/password
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   356
        if this account is defined in this source and valid login / password is
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   357
        given. Else raise `AuthenticationError`
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   358
        """
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   359
        raise NotImplementedError(self)
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   360
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   361
    # RQL query api ############################################################
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   362
9512
88dc96fc9fc1 [server] use a connection instead of a session for user authentication
Julien Cristau <julien.cristau@logilab.fr>
parents: 9469
diff changeset
   363
    def syntax_tree_search(self, cnx, union,
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   364
                           args=None, cachekey=None, varmap=None, debug=0):
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   365
        """return result from this source for a rql query (actually from a rql
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   366
        syntax tree and a solution dictionary mapping each used variable to a
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   367
        possible type). If cachekey is given, the query necessary to fetch the
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   368
        results (but not the results themselves) may be cached using this key.
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   369
        """
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   370
        raise NotImplementedError(self)
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   371
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   372
    # write modification api ###################################################
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   373
    # read-only sources don't have to implement methods below
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   374
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   375
    def get_extid(self, entity):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   376
        """return the external id for the given newly inserted entity"""
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   377
        raise NotImplementedError(self)
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   378
10086
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9822
diff changeset
   379
    def add_entity(self, cnx, entity):
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   380
        """add a new entity to the source"""
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   381
        raise NotImplementedError(self)
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   382
10086
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9822
diff changeset
   383
    def update_entity(self, cnx, entity):
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   384
        """update an entity in the source"""
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   385
        raise NotImplementedError(self)
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   386
10086
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9822
diff changeset
   387
    def delete_entities(self, cnx, entities):
6889
37668bf302f5 improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 6722
diff changeset
   388
        """delete several entities from the source"""
37668bf302f5 improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 6722
diff changeset
   389
        for entity in entities:
10086
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9822
diff changeset
   390
            self.delete_entity(cnx, entity)
6889
37668bf302f5 improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 6722
diff changeset
   391
10086
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9822
diff changeset
   392
    def delete_entity(self, cnx, entity):
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   393
        """delete an entity from the source"""
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   394
        raise NotImplementedError(self)
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   395
10086
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9822
diff changeset
   396
    def add_relation(self, cnx, subject, rtype, object):
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   397
        """add a relation to the source"""
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   398
        raise NotImplementedError(self)
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   399
10086
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9822
diff changeset
   400
    def add_relations(self, cnx,  rtype, subj_obj_list):
7237
9f619715665b [server] improve the speed of setting relations between entities (closes #1625257)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7118
diff changeset
   401
        """add a relations to the source"""
9f619715665b [server] improve the speed of setting relations between entities (closes #1625257)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7118
diff changeset
   402
        # override in derived classes if you feel you can
9f619715665b [server] improve the speed of setting relations between entities (closes #1625257)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7118
diff changeset
   403
        # optimize
9f619715665b [server] improve the speed of setting relations between entities (closes #1625257)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7118
diff changeset
   404
        for subject, object in subj_obj_list:
10086
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9822
diff changeset
   405
            self.add_relation(cnx, subject, rtype, object)
7237
9f619715665b [server] improve the speed of setting relations between entities (closes #1625257)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7118
diff changeset
   406
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   407
    def delete_relation(self, session, subject, rtype, object):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   408
        """delete a relation from the source"""
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   409
        raise NotImplementedError(self)
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   410
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   411
    # system source interface #################################################
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   412
10086
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9822
diff changeset
   413
    def eid_type_source(self, cnx, eid):
10575
036f5964e6fe [server/sources] fix docstring for eid_type_source
Julien Cristau <julien.cristau@logilab.fr>
parents: 10086
diff changeset
   414
        """return a tuple (type, extid, source) for the entity with id <eid>"""
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   415
        raise NotImplementedError(self)
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   416
10086
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9822
diff changeset
   417
    def create_eid(self, cnx):
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   418
        raise NotImplementedError(self)
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   419
10086
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9822
diff changeset
   420
    def add_info(self, cnx, entity, source, extid):
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   421
        """add type and source info for an eid into the system table"""
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   422
        raise NotImplementedError(self)
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   423
10086
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9822
diff changeset
   424
    def update_info(self, cnx, entity, need_fti_update):
4913
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 4807
diff changeset
   425
        """mark entity as being modified, fulltext reindex if needed"""
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   426
        raise NotImplementedError(self)
4913
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 4807
diff changeset
   427
10086
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9822
diff changeset
   428
    def index_entity(self, cnx, entity):
4807
5642bfa43236 [cleanup] add index_entity to abstract source, add docstring
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4252
diff changeset
   429
        """create an operation to [re]index textual content of the given entity
5642bfa43236 [cleanup] add index_entity to abstract source, add docstring
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4252
diff changeset
   430
        on commit
5642bfa43236 [cleanup] add index_entity to abstract source, add docstring
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4252
diff changeset
   431
        """
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   432
        raise NotImplementedError(self)
4807
5642bfa43236 [cleanup] add index_entity to abstract source, add docstring
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4252
diff changeset
   433
10086
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9822
diff changeset
   434
    def fti_unindex_entities(self, cnx, entities):
6889
37668bf302f5 improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 6722
diff changeset
   435
        """remove text content for entities from the full text index
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   436
        """
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   437
        raise NotImplementedError(self)
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   438
10086
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9822
diff changeset
   439
    def fti_index_entities(self, cnx, entities):
6889
37668bf302f5 improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 6722
diff changeset
   440
        """add text content of created/modified entities to the full text index
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   441
        """
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   442
        raise NotImplementedError(self)
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   443
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   444
    # sql system source interface #############################################
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   445
10086
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9822
diff changeset
   446
    def sqlexec(self, cnx, sql, args=None):
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   447
        """execute the query and return its result"""
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   448
        raise NotImplementedError(self)
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   449
10086
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9822
diff changeset
   450
    def create_index(self, cnx, table, column, unique=False):
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   451
        raise NotImplementedError(self)
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   452
10086
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9822
diff changeset
   453
    def drop_index(self, cnx, table, column, unique=False):
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   454
        raise NotImplementedError(self)
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   455
7697
ef50074a0314 [repo api] deprecates source.extid2eid, only the system source should implement it,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7552
diff changeset
   456
10086
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9822
diff changeset
   457
    @deprecated('[3.13] use extid2eid(source, value, etype, cnx, **kwargs)')
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9822
diff changeset
   458
    def extid2eid(self, value, etype, cnx, **kwargs):
98bc2ca1a816 [source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents: 9822
diff changeset
   459
        return self.repo.extid2eid(self, value, etype, cnx, **kwargs)
7697
ef50074a0314 [repo api] deprecates source.extid2eid, only the system source should implement it,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7552
diff changeset
   460
ef50074a0314 [repo api] deprecates source.extid2eid, only the system source should implement it,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7552
diff changeset
   461
6427
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6162
diff changeset
   462
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   463
6427
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6162
diff changeset
   464
def source_adapter(source_type):
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   465
    try:
6427
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6162
diff changeset
   466
        return SOURCE_TYPES[source_type]
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   467
    except KeyError:
6427
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6162
diff changeset
   468
        raise RuntimeError('Unknown source type %r' % source_type)
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   469
6945
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   470
def get_source(type, source_config, repo, eid):
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   471
    """return a source adapter according to the adapter field in the source's
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   472
    configuration
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   473
    """
6945
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   474
    return source_adapter(type)(repo, source_config, eid)