server/sources/__init__.py
author David Douard <david.douard@logilab.fr>
Mon, 10 Dec 2012 11:50:42 +0100
branchstable
changeset 8615 a24fbd24ede4
parent 8228 fe53c7e1625f
child 8544 3d049071957e
permissions -rw-r--r--
[ldapfeed] fix connection leak (closes #2532528)
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
6944
0cf10429ad39 [sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6943
diff changeset
     1
# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
5421
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5124
diff changeset
     2
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5124
diff changeset
     3
#
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5124
diff changeset
     4
# This file is part of CubicWeb.
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5124
diff changeset
     5
#
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5124
diff changeset
     6
# CubicWeb is free software: you can redistribute it and/or modify it under the
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5124
diff changeset
     7
# terms of the GNU Lesser General Public License as published by the Free
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5124
diff changeset
     8
# Software Foundation, either version 2.1 of the License, or (at your option)
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5124
diff changeset
     9
# any later version.
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5124
diff changeset
    10
#
5424
8ecbcbff9777 replace logilab-common by CubicWeb in disclaimer
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5421
diff changeset
    11
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
5421
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5124
diff changeset
    12
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5124
diff changeset
    13
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5124
diff changeset
    14
# details.
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5124
diff changeset
    15
#
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5124
diff changeset
    16
# You should have received a copy of the GNU Lesser General Public License along
8167de96c523 proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5124
diff changeset
    17
# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
5824
de9b7e88660e cleanup
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5642
diff changeset
    18
"""cubicweb server sources support"""
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    19
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    20
__docformat__ = "restructuredtext en"
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    21
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
    22
import itertools
2493
9806571ea790 major refactoring of database dump/restore:
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2476
diff changeset
    23
from os.path import join, splitext
1263
01152fffd593 backport default branch
sylvain.thenault@logilab.fr
parents: 1251
diff changeset
    24
from datetime import datetime, timedelta
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    25
from logging import getLogger
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    26
6945
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
    27
from logilab.common import configuration
7697
ef50074a0314 [repo api] deprecates source.extid2eid, only the system source should implement it,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7552
diff changeset
    28
from logilab.common.deprecation import deprecated
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    29
6945
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
    30
from yams.schema import role_name
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
    31
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
    32
from cubicweb import ValidationError, set_log_methods, server
2596
d02eed70937f [R repo, schema] use VIRTUAL_RTYPES const
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2493
diff changeset
    33
from cubicweb.schema import VIRTUAL_RTYPES
1251
af40e615dc89 introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents: 1250
diff changeset
    34
from cubicweb.server.sqlutils import SQL_PREFIX
7118
e094b3d4eb95 [server] move EditedEntity class to its own module, to avoid cyclic dependency when needed from e.g. session.py
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7040
diff changeset
    35
from cubicweb.server.edition import EditedEntity
1251
af40e615dc89 introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents: 1250
diff changeset
    36
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    37
2625
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2596
diff changeset
    38
def dbg_st_search(uri, union, varmap, args, cachekey=None, prefix='rql for'):
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2596
diff changeset
    39
    if server.DEBUG & server.DBG_RQL:
7779
3826d8480a68 [debug] use repr() in some place where you'll have a chance to get an unicode error if output encoding isn't properly detected
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7697
diff changeset
    40
        print '  %s %s source: %s' % (prefix, uri, repr(union.as_string()))
2625
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2596
diff changeset
    41
        if varmap:
2719
6df328b47361 [server debug] indent for more readability
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2650
diff changeset
    42
            print '    using varmap', varmap
2625
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2596
diff changeset
    43
        if server.DEBUG & server.DBG_MORE:
7779
3826d8480a68 [debug] use repr() in some place where you'll have a chance to get an unicode error if output encoding isn't properly detected
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7697
diff changeset
    44
            print '    args', repr(args)
2719
6df328b47361 [server debug] indent for more readability
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2650
diff changeset
    45
            print '    cache key', cachekey
6df328b47361 [server debug] indent for more readability
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2650
diff changeset
    46
            print '    solutions', ','.join(str(s.solutions)
6df328b47361 [server debug] indent for more readability
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2650
diff changeset
    47
                                            for s in union.children)
2625
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2596
diff changeset
    48
    # return true so it can be used as assertion (and so be killed by python -O)
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2596
diff changeset
    49
    return True
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2596
diff changeset
    50
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2596
diff changeset
    51
def dbg_results(results):
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2596
diff changeset
    52
    if server.DEBUG & server.DBG_RQL:
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2596
diff changeset
    53
        if len(results) > 10:
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2596
diff changeset
    54
            print '  -->', results[:10], '...', len(results)
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2596
diff changeset
    55
        else:
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2596
diff changeset
    56
            print '  -->', results
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2596
diff changeset
    57
    # return true so it can be used as assertion (and so be killed by python -O)
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2596
diff changeset
    58
    return True
2493
9806571ea790 major refactoring of database dump/restore:
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2476
diff changeset
    59
1238
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 386
diff changeset
    60
class TimedCache(dict):
5642
6a90357b9769 TimedCache now only accepts values expressed in seconds
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 5637
diff changeset
    61
    def __init__(self, ttl):
6a90357b9769 TimedCache now only accepts values expressed in seconds
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 5637
diff changeset
    62
        # time to live in seconds
6a90357b9769 TimedCache now only accepts values expressed in seconds
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 5637
diff changeset
    63
        if ttl <= 0:
6582
8eb7883b4223 [pylint] fix a bug of pylint detected errors and i18n pb (calling builtins._ instead of req._)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6427
diff changeset
    64
            raise ValueError('TimedCache initialized with a ttl of %ss' % ttl.seconds)
5642
6a90357b9769 TimedCache now only accepts values expressed in seconds
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 5637
diff changeset
    65
        self.ttl = timedelta(seconds=ttl)
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
    66
1238
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 386
diff changeset
    67
    def __setitem__(self, key, value):
7922
d307c3817782 [repository] use utcnow instead of now for some internal timetamps (closes #1988458)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7879
diff changeset
    68
        dict.__setitem__(self, key, (datetime.utcnow(), value))
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
    69
1238
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 386
diff changeset
    70
    def __getitem__(self, key):
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 386
diff changeset
    71
        return dict.__getitem__(self, key)[1]
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
    72
1238
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 386
diff changeset
    73
    def clear_expired(self):
7922
d307c3817782 [repository] use utcnow instead of now for some internal timetamps (closes #1988458)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7879
diff changeset
    74
        now_ = datetime.utcnow()
1238
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 386
diff changeset
    75
        ttl = self.ttl
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 386
diff changeset
    76
        for key, (timestamp, value) in self.items():
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 386
diff changeset
    77
            if now_ - timestamp > ttl:
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 386
diff changeset
    78
                del self[key]
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 386
diff changeset
    79
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    80
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    81
class AbstractSource(object):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    82
    """an abstract class for sources"""
6957
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
    83
    # does the source copy data into the system source, or is it a *true* source
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
    84
    # (i.e. entities are not stored physically here)
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
    85
    copy_based_source = False
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    86
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    87
    # boolean telling if modification hooks should be called when something is
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    88
    # modified in this source
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    89
    should_call_hooks = True
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    90
    # boolean telling if the repository should connect to this source during
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    91
    # migration
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    92
    connect_for_migration = True
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
    93
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    94
    # mappings telling which entities and relations are available in the source
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    95
    # keys are supported entity/relation types and values are boolean indicating
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    96
    # wether the support is read-only (False) or read-write (True)
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    97
    support_entities = {}
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    98
    support_relations = {}
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
    99
    # a global identifier for this source, which has to be set by the source
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   100
    # instance
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   101
    uri = None
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   102
    # a reference to the system information helper
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   103
    repo = None
2476
1294a6bdf3bf application -> instance where it makes sense
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1977
diff changeset
   104
    # a reference to the instance'schema (may differs from the source'schema)
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   105
    schema = None
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   106
3040
e4dc22040f5f [multi-sources] move [dont_]cross_relations to AbstractSource
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2759
diff changeset
   107
    # multi-sources planning control
e4dc22040f5f [multi-sources] move [dont_]cross_relations to AbstractSource
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2759
diff changeset
   108
    dont_cross_relations = ()
e4dc22040f5f [multi-sources] move [dont_]cross_relations to AbstractSource
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2759
diff changeset
   109
    cross_relations = ()
e4dc22040f5f [multi-sources] move [dont_]cross_relations to AbstractSource
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2759
diff changeset
   110
6427
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6162
diff changeset
   111
    # force deactivation (configuration error for instance)
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6162
diff changeset
   112
    disabled = False
3040
e4dc22040f5f [multi-sources] move [dont_]cross_relations to AbstractSource
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2759
diff changeset
   113
7552
82dde8276a5b [datafeed, entities] url for entities from a datafeed source should be on their origin site. Closes #1769391
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7514
diff changeset
   114
    # boolean telling if cwuri of entities from this source is the url that
82dde8276a5b [datafeed, entities] url for entities from a datafeed source should be on their origin site. Closes #1769391
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7514
diff changeset
   115
    # should be used as entity's absolute url
82dde8276a5b [datafeed, entities] url for entities from a datafeed source should be on their origin site. Closes #1769391
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7514
diff changeset
   116
    use_cwuri_as_url = False
82dde8276a5b [datafeed, entities] url for entities from a datafeed source should be on their origin site. Closes #1769391
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7514
diff changeset
   117
6945
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   118
    # source configuration options
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   119
    options = ()
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   120
7879
9aae456abab5 [pylint] fix pylint detected errors and tweak it so that pylint -E will be much less verbose next time (+ update some copyrights on the way)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7815
diff changeset
   121
    # these are overridden by set_log_methods below
9aae456abab5 [pylint] fix pylint detected errors and tweak it so that pylint -E will be much less verbose next time (+ update some copyrights on the way)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7815
diff changeset
   122
    # only defining here to prevent pylint from complaining
9aae456abab5 [pylint] fix pylint detected errors and tweak it so that pylint -E will be much less verbose next time (+ update some copyrights on the way)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7815
diff changeset
   123
    info = warning = error = critical = exception = debug = lambda msg,*a,**kw: None
9aae456abab5 [pylint] fix pylint detected errors and tweak it so that pylint -E will be much less verbose next time (+ update some copyrights on the way)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7815
diff changeset
   124
6945
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   125
    def __init__(self, repo, source_config, eid=None):
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   126
        self.repo = repo
6427
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6162
diff changeset
   127
        self.set_schema(repo.schema)
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   128
        self.support_relations['identity'] = False
6945
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   129
        self.eid = eid
6722
3341521d857b [repo source] rename attribute so it's much easier to grasp its role
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6582
diff changeset
   130
        self.public_config = source_config.copy()
7552
82dde8276a5b [datafeed, entities] url for entities from a datafeed source should be on their origin site. Closes #1769391
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7514
diff changeset
   131
        self.public_config.setdefault('use-cwuri-as-url', self.use_cwuri_as_url)
6722
3341521d857b [repo source] rename attribute so it's much easier to grasp its role
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6582
diff changeset
   132
        self.remove_sensitive_information(self.public_config)
6945
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   133
        self.uri = source_config.pop('uri')
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   134
        set_log_methods(self, getLogger('cubicweb.sources.'+self.uri))
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   135
        source_config.pop('type')
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   136
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   137
    def __repr__(self):
8228
fe53c7e1625f [sources] include source's class in its repr
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8134
diff changeset
   138
        return '<%s %s source %s @%#x>' % (self.uri, self.__class__.__name__,
fe53c7e1625f [sources] include source's class in its repr
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 8134
diff changeset
   139
                                           self.eid, id(self))
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   140
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   141
    def __cmp__(self, other):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   142
        """simple comparison function to get predictable source order, with the
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   143
        system source at last
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   144
        """
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   145
        if self.uri == other.uri:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   146
            return 0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   147
        if self.uri == 'system':
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   148
            return 1
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   149
        if other.uri == 'system':
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   150
            return -1
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   151
        return cmp(self.uri, other.uri)
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   152
7342
d1c8b5b3531c adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7237
diff changeset
   153
    def backup(self, backupfile, confirm, format='native'):
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   154
        """method called to create a backup of source's data"""
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   155
        pass
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   156
7342
d1c8b5b3531c adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7237
diff changeset
   157
    def restore(self, backupfile, confirm, drop, format='native'):
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   158
        """method called to restore a backup of source's data"""
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   159
        pass
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   160
6945
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   161
    @classmethod
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   162
    def check_conf_dict(cls, eid, confdict, _=unicode, fail_if_unknown=True):
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   163
        """check configuration of source entity. Return config dict properly
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   164
        typed with defaults set.
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   165
        """
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   166
        processed = {}
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   167
        for optname, optdict in cls.options:
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   168
            value = confdict.pop(optname, optdict.get('default'))
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   169
            if value is configuration.REQUIRED:
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   170
                if not fail_if_unknown:
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   171
                    continue
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   172
                msg = _('specifying %s is mandatory' % optname)
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   173
                raise ValidationError(eid, {role_name('config', 'subject'): msg})
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   174
            elif value is not None:
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   175
                # type check
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   176
                try:
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   177
                    value = configuration.convert(value, optdict, optname)
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   178
                except Exception, ex:
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   179
                    msg = unicode(ex) # XXX internationalization
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   180
                    raise ValidationError(eid, {role_name('config', 'subject'): msg})
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   181
            processed[optname] = value
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   182
        # cw < 3.10 bw compat
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   183
        try:
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   184
            processed['adapter'] = confdict['adapter']
7815
2a164a9cf81c [exceptions] stop catching any exception in various places (closes #1942716)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7779
diff changeset
   185
        except KeyError:
6945
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   186
            pass
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   187
        # check for unknown options
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   188
        if confdict and not confdict.keys() == ['adapter']:
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   189
            if fail_if_unknown:
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   190
                msg = _('unknown options %s') % ', '.join(confdict)
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   191
                raise ValidationError(eid, {role_name('config', 'subject'): msg})
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   192
            else:
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   193
                logger = getLogger('cubicweb.sources')
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   194
                logger.warning('unknown options %s', ', '.join(confdict))
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   195
                # add options to processed, they may be necessary during migration
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   196
                processed.update(confdict)
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   197
        return processed
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   198
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   199
    @classmethod
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   200
    def check_config(cls, source_entity):
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   201
        """check configuration of source entity"""
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   202
        return cls.check_conf_dict(source_entity.eid, source_entity.host_config,
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   203
                                    _=source_entity._cw._)
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   204
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   205
    def update_config(self, source_entity, typedconfig):
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   206
        """update configuration from source entity. `typedconfig` is config
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   207
        properly typed with defaults set
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   208
        """
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   209
        pass
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   210
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   211
    # source initialization / finalization #####################################
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   212
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   213
    def set_schema(self, schema):
2476
1294a6bdf3bf application -> instance where it makes sense
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1977
diff changeset
   214
        """set the instance'schema"""
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   215
        self.schema = schema
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   216
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   217
    def init_creating(self):
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   218
        """method called by the repository once ready to create a new instance"""
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   219
        pass
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   220
6957
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
   221
    def init(self, activated, source_entity):
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   222
        """method called by the repository once ready to handle request.
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   223
        `activated` is a boolean flag telling if the source is activated or not.
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   224
        """
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   225
        pass
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   226
7552
82dde8276a5b [datafeed, entities] url for entities from a datafeed source should be on their origin site. Closes #1769391
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7514
diff changeset
   227
    PUBLIC_KEYS = ('type', 'uri', 'use-cwuri-as-url')
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   228
    def remove_sensitive_information(self, sourcedef):
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   229
        """remove sensitive information such as login / password from source
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   230
        definition
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   231
        """
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   232
        for key in sourcedef.keys():
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   233
            if not key in self.PUBLIC_KEYS:
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   234
                sourcedef.pop(key)
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   235
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   236
    # connections handling #####################################################
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   237
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   238
    def get_connection(self):
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   239
        """open and return a connection to the source"""
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   240
        raise NotImplementedError(self)
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   241
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   242
    def check_connection(self, cnx):
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   243
        """Check connection validity, return None if the connection is still
7398
26695dd703d8 [repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7342
diff changeset
   244
        valid else a new connection (called when the connections set using the
26695dd703d8 [repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7342
diff changeset
   245
        given connection is being attached to a session). Do nothing by default.
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   246
        """
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   247
        pass
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   248
7398
26695dd703d8 [repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7342
diff changeset
   249
    def close_source_connections(self):
26695dd703d8 [repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7342
diff changeset
   250
        for cnxset in self.repo.cnxsets:
26695dd703d8 [repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7342
diff changeset
   251
            cnxset._cursors.pop(self.uri, None)
26695dd703d8 [repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7342
diff changeset
   252
            cnxset.source_cnxs[self.uri][1].close()
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   253
7398
26695dd703d8 [repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7342
diff changeset
   254
    def open_source_connections(self):
26695dd703d8 [repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7342
diff changeset
   255
        for cnxset in self.repo.cnxsets:
26695dd703d8 [repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7342
diff changeset
   256
            cnxset.source_cnxs[self.uri] = (self, self.get_connection())
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   257
7398
26695dd703d8 [repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7342
diff changeset
   258
    def cnxset_freed(self, cnx):
26695dd703d8 [repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7342
diff changeset
   259
        """the connections set holding the given connection is being reseted
26695dd703d8 [repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7342
diff changeset
   260
        from its current attached session.
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   261
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   262
        do nothing by default
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   263
        """
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   264
        pass
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   265
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   266
    # cache handling ###########################################################
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   267
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   268
    def reset_caches(self):
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   269
        """method called during test to reset potential source caches"""
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   270
        pass
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   271
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   272
    def clear_eid_cache(self, eid, etype):
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   273
        """clear potential caches for the given eid"""
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   274
        pass
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   275
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   276
    # external source api ######################################################
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   277
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   278
    def support_entity(self, etype, write=False):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   279
        """return true if the given entity's type is handled by this adapter
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   280
        if write is true, return true only if it's a RW support
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   281
        """
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   282
        try:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   283
            wsupport = self.support_entities[etype]
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   284
        except KeyError:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   285
            return False
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   286
        if write:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   287
            return wsupport
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   288
        return True
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   289
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   290
    def support_relation(self, rtype, write=False):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   291
        """return true if the given relation's type is handled by this adapter
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   292
        if write is true, return true only if it's a RW support
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   293
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   294
        current implementation return true if the relation is defined into
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   295
        `support_relations` or if it is a final relation of a supported entity
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   296
        type
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   297
        """
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   298
        try:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   299
            wsupport = self.support_relations[rtype]
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   300
        except KeyError:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   301
            rschema = self.schema.rschema(rtype)
6057
cb5c8852cbda [ms] cleanups
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5849
diff changeset
   302
            if not rschema.final or rschema.type == 'has_text':
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   303
                return False
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   304
            for etype in rschema.subjects():
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   305
                try:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   306
                    wsupport = self.support_entities[etype]
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   307
                    break
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   308
                except KeyError:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   309
                    continue
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   310
            else:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   311
                return False
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   312
        if write:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   313
            return wsupport
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   314
        return True
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   315
3041
782fa7566a22 [multi-sources] new may_cross_relation method on sources
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3040
diff changeset
   316
    def may_cross_relation(self, rtype):
782fa7566a22 [multi-sources] new may_cross_relation method on sources
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3040
diff changeset
   317
        """return True if the relation may be crossed among sources. Rules are:
782fa7566a22 [multi-sources] new may_cross_relation method on sources
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3040
diff changeset
   318
782fa7566a22 [multi-sources] new may_cross_relation method on sources
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3040
diff changeset
   319
        * if this source support the relation, can't be crossed unless explicitly
782fa7566a22 [multi-sources] new may_cross_relation method on sources
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3040
diff changeset
   320
          specified in .cross_relations
782fa7566a22 [multi-sources] new may_cross_relation method on sources
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3040
diff changeset
   321
782fa7566a22 [multi-sources] new may_cross_relation method on sources
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3040
diff changeset
   322
        * if this source doesn't support the relation, can be crossed unless
782fa7566a22 [multi-sources] new may_cross_relation method on sources
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3040
diff changeset
   323
          explicitly specified in .dont_cross_relations
782fa7566a22 [multi-sources] new may_cross_relation method on sources
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3040
diff changeset
   324
        """
3633
2d81178bea5f [multi-source] add test case for an encountered bug, fixed by adding state_of to dont_cross_relations. Added a note to remind me that something should be done to be smarter in handling this
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3041
diff changeset
   325
        # XXX find a way to have relation such as state_of in dont cross
2d81178bea5f [multi-source] add test case for an encountered bug, fixed by adding state_of to dont_cross_relations. Added a note to remind me that something should be done to be smarter in handling this
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3041
diff changeset
   326
        #     relation (eg composite relation without both end type available?
2d81178bea5f [multi-source] add test case for an encountered bug, fixed by adding state_of to dont_cross_relations. Added a note to remind me that something should be done to be smarter in handling this
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3041
diff changeset
   327
        #     card 1 relation ? ...)
3041
782fa7566a22 [multi-sources] new may_cross_relation method on sources
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3040
diff changeset
   328
        if self.support_relation(rtype):
782fa7566a22 [multi-sources] new may_cross_relation method on sources
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3040
diff changeset
   329
            return rtype in self.cross_relations
782fa7566a22 [multi-sources] new may_cross_relation method on sources
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3040
diff changeset
   330
        return rtype not in self.dont_cross_relations
782fa7566a22 [multi-sources] new may_cross_relation method on sources
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3040
diff changeset
   331
6957
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
   332
    def before_entity_insertion(self, session, lid, etype, eid, sourceparams):
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   333
        """called by the repository when an eid has been attributed for an
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   334
        entity stored here but the entity has not been inserted in the system
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   335
        table yet.
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   336
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   337
        This method must return the an Entity instance representation of this
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   338
        entity.
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   339
        """
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   340
        entity = self.repo.vreg['etypes'].etype_class(etype)(session)
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   341
        entity.eid = eid
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   342
        entity.cw_edited = EditedEntity(entity)
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   343
        return entity
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   344
6957
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
   345
    def after_entity_insertion(self, session, lid, entity, sourceparams):
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   346
        """called by the repository after an entity stored here has been
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   347
        inserted in the system table.
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   348
        """
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   349
        pass
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   350
6957
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
   351
    def _load_mapping(self, session=None, **kwargs):
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
   352
        if not 'CWSourceSchemaConfig' in self.schema:
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
   353
            self.warning('instance is not mapping ready')
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
   354
            return
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
   355
        if session is None:
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
   356
            _session = self.repo.internal_session()
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
   357
        else:
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
   358
            _session = session
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
   359
        try:
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
   360
            for schemacfg in _session.execute(
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
   361
                'Any CFG,CFGO,S WHERE '
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
   362
                'CFG options CFGO, CFG cw_schema S, '
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
   363
                'CFG cw_for_source X, X eid %(x)s', {'x': self.eid}).entities():
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
   364
                self.add_schema_config(schemacfg, **kwargs)
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
   365
        finally:
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
   366
            if session is None:
ffda12be2e9f [repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6945
diff changeset
   367
                _session.close()
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   368
6944
0cf10429ad39 [sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6943
diff changeset
   369
    def add_schema_config(self, schemacfg, checkonly=False):
0cf10429ad39 [sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6943
diff changeset
   370
        """added CWSourceSchemaConfig, modify mapping accordingly"""
0cf10429ad39 [sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6943
diff changeset
   371
        msg = schemacfg._cw._("this source doesn't use a mapping")
0cf10429ad39 [sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6943
diff changeset
   372
        raise ValidationError(schemacfg.eid, {None: msg})
0cf10429ad39 [sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6943
diff changeset
   373
0cf10429ad39 [sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6943
diff changeset
   374
    def del_schema_config(self, schemacfg, checkonly=False):
0cf10429ad39 [sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6943
diff changeset
   375
        """deleted CWSourceSchemaConfig, modify mapping accordingly"""
0cf10429ad39 [sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6943
diff changeset
   376
        msg = schemacfg._cw._("this source doesn't use a mapping")
0cf10429ad39 [sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6943
diff changeset
   377
        raise ValidationError(schemacfg.eid, {None: msg})
0cf10429ad39 [sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6943
diff changeset
   378
0cf10429ad39 [sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6943
diff changeset
   379
    def update_schema_config(self, schemacfg, checkonly=False):
0cf10429ad39 [sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6943
diff changeset
   380
        """updated CWSourceSchemaConfig, modify mapping accordingly"""
0cf10429ad39 [sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6943
diff changeset
   381
        self.del_schema_config(schemacfg, checkonly)
0cf10429ad39 [sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6943
diff changeset
   382
        self.add_schema_config(schemacfg, checkonly)
0cf10429ad39 [sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6943
diff changeset
   383
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   384
    # user authentication api ##################################################
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   385
3647
2941f4a0aab9 refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3041
diff changeset
   386
    def authenticate(self, session, login, **kwargs):
6152
6824f8b61098 use is_instance in a number of places (esp. documentation) rather than the deprecated implements
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 6057
diff changeset
   387
        """if the source support CWUser entity type, it should implement
1398
5fe84a5f7035 rename internal entity types to have CW prefix instead of E
sylvain.thenault@logilab.fr
parents: 1263
diff changeset
   388
        this method which should return CWUser eid for the given login/password
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   389
        if this account is defined in this source and valid login / password is
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   390
        given. Else raise `AuthenticationError`
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   391
        """
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   392
        raise NotImplementedError(self)
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   393
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   394
    # RQL query api ############################################################
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   395
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   396
    def syntax_tree_search(self, session, union,
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   397
                           args=None, cachekey=None, varmap=None, debug=0):
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   398
        """return result from this source for a rql query (actually from a rql
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   399
        syntax tree and a solution dictionary mapping each used variable to a
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   400
        possible type). If cachekey is given, the query necessary to fetch the
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   401
        results (but not the results themselves) may be cached using this key.
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   402
        """
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   403
        raise NotImplementedError(self)
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   404
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   405
    def flying_insert(self, table, session, union, args=None, varmap=None):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   406
        """similar as .syntax_tree_search, but inserts data in the temporary
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   407
        table (on-the-fly if possible, eg for the system source whose the given
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   408
        cursor come from). If not possible, inserts all data by calling
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   409
        .executemany().
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   410
        """
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   411
        res = self.syntax_tree_search(session, union, args, varmap=varmap)
7398
26695dd703d8 [repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7342
diff changeset
   412
        session.cnxset.source('system').manual_insert(res, table, session)
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   413
6943
406a41c25e13 [sources] reorganize abstract source code, drop no more used cleanup_entities_info and _cleanup_system_relations methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6931
diff changeset
   414
    # write modification api ###################################################
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   415
    # read-only sources don't have to implement methods below
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   416
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   417
    def get_extid(self, entity):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   418
        """return the external id for the given newly inserted entity"""
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   419
        raise NotImplementedError(self)
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   420
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   421
    def add_entity(self, session, entity):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   422
        """add a new entity to the source"""
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   423
        raise NotImplementedError(self)
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   424
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   425
    def update_entity(self, session, entity):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   426
        """update an entity in the source"""
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   427
        raise NotImplementedError(self)
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   428
6889
37668bf302f5 improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 6722
diff changeset
   429
    def delete_entities(self, session, entities):
37668bf302f5 improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 6722
diff changeset
   430
        """delete several entities from the source"""
37668bf302f5 improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 6722
diff changeset
   431
        for entity in entities:
37668bf302f5 improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 6722
diff changeset
   432
            self.delete_entity(session, entity)
37668bf302f5 improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 6722
diff changeset
   433
4913
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 4807
diff changeset
   434
    def delete_entity(self, session, entity):
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   435
        """delete an entity from the source"""
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   436
        raise NotImplementedError(self)
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   437
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   438
    def add_relation(self, session, subject, rtype, object):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   439
        """add a relation to the source"""
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   440
        raise NotImplementedError(self)
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   441
7237
9f619715665b [server] improve the speed of setting relations between entities (closes #1625257)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7118
diff changeset
   442
    def add_relations(self, session,  rtype, subj_obj_list):
9f619715665b [server] improve the speed of setting relations between entities (closes #1625257)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7118
diff changeset
   443
        """add a relations to the source"""
9f619715665b [server] improve the speed of setting relations between entities (closes #1625257)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7118
diff changeset
   444
        # override in derived classes if you feel you can
9f619715665b [server] improve the speed of setting relations between entities (closes #1625257)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7118
diff changeset
   445
        # optimize
9f619715665b [server] improve the speed of setting relations between entities (closes #1625257)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7118
diff changeset
   446
        for subject, object in subj_obj_list:
9f619715665b [server] improve the speed of setting relations between entities (closes #1625257)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7118
diff changeset
   447
            self.add_relation(session, subject, rtype, object)
9f619715665b [server] improve the speed of setting relations between entities (closes #1625257)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 7118
diff changeset
   448
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   449
    def delete_relation(self, session, subject, rtype, object):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   450
        """delete a relation from the source"""
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   451
        raise NotImplementedError(self)
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   452
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   453
    # system source interface #################################################
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   454
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   455
    def eid_type_source(self, session, eid):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   456
        """return a tuple (type, source, extid) for the entity with id <eid>"""
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   457
        raise NotImplementedError(self)
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   458
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   459
    def create_eid(self, session):
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   460
        raise NotImplementedError(self)
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   461
4913
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 4807
diff changeset
   462
    def add_info(self, session, entity, source, extid):
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   463
        """add type and source info for an eid into the system table"""
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   464
        raise NotImplementedError(self)
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   465
4913
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 4807
diff changeset
   466
    def update_info(self, session, entity, need_fti_update):
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 4807
diff changeset
   467
        """mark entity as being modified, fulltext reindex if needed"""
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   468
        raise NotImplementedError(self)
4913
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 4807
diff changeset
   469
7501
2983dd24494a [repository] refactor/cleanup entity deletion methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7342
diff changeset
   470
    def delete_info_multi(self, session, entities, uri):
2983dd24494a [repository] refactor/cleanup entity deletion methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7342
diff changeset
   471
        """delete system information on deletion of a list of entities with the
2983dd24494a [repository] refactor/cleanup entity deletion methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7342
diff changeset
   472
        same etype and belinging to the same source
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   473
        """
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   474
        raise NotImplementedError(self)
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   475
4807
5642bfa43236 [cleanup] add index_entity to abstract source, add docstring
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4252
diff changeset
   476
    def modified_entities(self, session, etypes, mtime):
5642bfa43236 [cleanup] add index_entity to abstract source, add docstring
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4252
diff changeset
   477
        """return a 2-uple:
5642bfa43236 [cleanup] add index_entity to abstract source, add docstring
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4252
diff changeset
   478
        * list of (etype, eid) of entities of the given types which have been
5642bfa43236 [cleanup] add index_entity to abstract source, add docstring
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4252
diff changeset
   479
          modified since the given timestamp (actually entities whose full text
5642bfa43236 [cleanup] add index_entity to abstract source, add docstring
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4252
diff changeset
   480
          index content has changed)
5642bfa43236 [cleanup] add index_entity to abstract source, add docstring
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4252
diff changeset
   481
        * list of (etype, eid) of entities of the given types which have been
5642bfa43236 [cleanup] add index_entity to abstract source, add docstring
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4252
diff changeset
   482
          deleted since the given timestamp
5642bfa43236 [cleanup] add index_entity to abstract source, add docstring
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4252
diff changeset
   483
        """
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   484
        raise NotImplementedError(self)
4807
5642bfa43236 [cleanup] add index_entity to abstract source, add docstring
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4252
diff changeset
   485
5642bfa43236 [cleanup] add index_entity to abstract source, add docstring
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4252
diff changeset
   486
    def index_entity(self, session, entity):
5642bfa43236 [cleanup] add index_entity to abstract source, add docstring
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4252
diff changeset
   487
        """create an operation to [re]index textual content of the given entity
5642bfa43236 [cleanup] add index_entity to abstract source, add docstring
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4252
diff changeset
   488
        on commit
5642bfa43236 [cleanup] add index_entity to abstract source, add docstring
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4252
diff changeset
   489
        """
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   490
        raise NotImplementedError(self)
4807
5642bfa43236 [cleanup] add index_entity to abstract source, add docstring
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4252
diff changeset
   491
6889
37668bf302f5 improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 6722
diff changeset
   492
    def fti_unindex_entities(self, session, entities):
37668bf302f5 improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 6722
diff changeset
   493
        """remove text content for entities from the full text index
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   494
        """
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   495
        raise NotImplementedError(self)
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   496
6889
37668bf302f5 improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 6722
diff changeset
   497
    def fti_index_entities(self, session, entities):
37668bf302f5 improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents: 6722
diff changeset
   498
        """add text content of created/modified entities to the full text index
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   499
        """
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   500
        raise NotImplementedError(self)
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   501
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   502
    # sql system source interface #############################################
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   503
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   504
    def sqlexec(self, session, sql, args=None):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   505
        """execute the query and return its result"""
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   506
        raise NotImplementedError(self)
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   507
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   508
    def temp_table_def(self, selection, solution, table, basemap):
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   509
        raise NotImplementedError(self)
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   510
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   511
    def create_index(self, session, table, column, unique=False):
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   512
        raise NotImplementedError(self)
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   513
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   514
    def drop_index(self, session, table, column, unique=False):
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   515
        raise NotImplementedError(self)
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   516
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   517
    def create_temp_table(self, session, table, schema):
8134
7f93da785e3a add instance to not implemented error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7922
diff changeset
   518
        raise NotImplementedError(self)
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   519
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   520
    def clean_temp_data(self, session, temptables):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   521
        """remove temporary data, usually associated to temporary tables"""
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   522
        pass
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   523
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   524
7697
ef50074a0314 [repo api] deprecates source.extid2eid, only the system source should implement it,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7552
diff changeset
   525
    @deprecated('[3.13] use repo.eid2extid(source, eid, session)')
ef50074a0314 [repo api] deprecates source.extid2eid, only the system source should implement it,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7552
diff changeset
   526
    def eid2extid(self, eid, session=None):
ef50074a0314 [repo api] deprecates source.extid2eid, only the system source should implement it,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7552
diff changeset
   527
        return self.repo.eid2extid(self, eid, session)
ef50074a0314 [repo api] deprecates source.extid2eid, only the system source should implement it,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7552
diff changeset
   528
ef50074a0314 [repo api] deprecates source.extid2eid, only the system source should implement it,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7552
diff changeset
   529
    @deprecated('[3.13] use extid2eid(source, value, etype, session, **kwargs)')
ef50074a0314 [repo api] deprecates source.extid2eid, only the system source should implement it,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7552
diff changeset
   530
    def extid2eid(self, value, etype, session=None, **kwargs):
ef50074a0314 [repo api] deprecates source.extid2eid, only the system source should implement it,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7552
diff changeset
   531
        return self.repo.extid2eid(self, value, etype, session, **kwargs)
ef50074a0314 [repo api] deprecates source.extid2eid, only the system source should implement it,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7552
diff changeset
   532
ef50074a0314 [repo api] deprecates source.extid2eid, only the system source should implement it,
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 7552
diff changeset
   533
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   534
class TrFunc(object):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   535
    """lower, upper"""
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   536
    def __init__(self, trname, index, attrname=None):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   537
        self._tr = trname.lower()
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   538
        self.index = index
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   539
        self.attrname = attrname
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   540
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   541
    def apply(self, resdict):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   542
        value = resdict.get(self.attrname)
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   543
        if value is not None:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   544
            return getattr(value, self._tr)()
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   545
        return None
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   546
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   547
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   548
class GlobTrFunc(TrFunc):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   549
    """count, sum, max, min, avg"""
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   550
    funcs = {
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   551
        'count': len,
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   552
        'sum': sum,
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   553
        'max': max,
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   554
        'min': min,
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   555
        # XXX avg
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   556
        }
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   557
    def apply(self, result):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   558
        """have to 'groupby' manually. For instance, if we 'count' for index 1:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   559
        >>> self.apply([(1, 2), (3, 4), (1, 5)])
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   560
        [(1, 7), (3, 4)]
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   561
        """
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   562
        keys, values = [], {}
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   563
        for row in result:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   564
            key = tuple(v for i, v in enumerate(row) if i != self.index)
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   565
            value = row[self.index]
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   566
            try:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   567
                values[key].append(value)
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   568
            except KeyError:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   569
                keys.append(key)
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   570
                values[key] = [value]
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   571
        result = []
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   572
        trfunc = self.funcs[self._tr]
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   573
        for key in keys:
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   574
            row = list(key)
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   575
            row.insert(self.index, trfunc(values[key]))
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   576
            result.append(row)
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   577
        return result
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   578
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   579
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   580
class ConnectionWrapper(object):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   581
    def __init__(self, cnx=None):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   582
        self.cnx = cnx
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   583
    def commit(self):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   584
        pass
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   585
    def rollback(self):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   586
        pass
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   587
    def cursor(self):
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   588
        return None # no actual cursor support
7037
6a8235456fe1 [multi-sources] add missing close method on ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6889
diff changeset
   589
    def close(self):
6a8235456fe1 [multi-sources] add missing close method on ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6889
diff changeset
   590
        if hasattr(self.cnx, 'close'):
6a8235456fe1 [multi-sources] add missing close method on ConnectionWrapper
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6889
diff changeset
   591
            self.cnx.close()
6427
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6162
diff changeset
   592
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   593
from cubicweb.server import SOURCE_TYPES
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   594
6427
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6162
diff changeset
   595
def source_adapter(source_type):
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   596
    try:
6427
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6162
diff changeset
   597
        return SOURCE_TYPES[source_type]
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   598
    except KeyError:
6427
c8a5ac2d1eaa [schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6162
diff changeset
   599
        raise RuntimeError('Unknown source type %r' % source_type)
1792
9eadf34fd860 delete-trailing-spaces
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   600
6945
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   601
def get_source(type, source_config, repo, eid):
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   602
    """return a source adapter according to the adapter field in the source's
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   603
    configuration
0
b97547f5f1fa Showtime !
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
diff changeset
   604
    """
6945
28bf94d062a9 [sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 6944
diff changeset
   605
    return source_adapter(type)(repo, source_config, eid)