server/sources/pyrorql.py
author Sylvain Thénault <sylvain.thenault@logilab.fr>
Thu, 21 Jan 2010 10:52:24 +0100
changeset 4308 ab3f1520502c
parent 4212 ab6573088b4a
child 4816 c02583cb80a9
permissions -rw-r--r--
this clutter startup screen for little benefit, remove it
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     1
"""Source to query another RQL repository using pyro
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     2
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     3
:organization: Logilab
4212
ab6573088b4a update copyright: welcome 2010
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents: 3689
diff changeset
     4
:copyright: 2007-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2.
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     5
:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr
1977
606923dff11b big bunch of copyright / docstring update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents: 1954
diff changeset
     6
:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     7
"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     8
__docformat__ = "restructuredtext en"
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     9
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    10
import threading
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    11
from os.path import join
1016
26387b836099 use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents: 520
diff changeset
    12
from time import mktime
26387b836099 use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents: 520
diff changeset
    13
from datetime import datetime
1952
8e19c813750d fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1805
diff changeset
    14
from base64 import b64decode
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    15
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    16
from Pyro.errors import PyroError, ConnectionClosedError
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    17
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    18
from logilab.common.configuration import REQUIRED
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    19
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    20
from rql.nodes import Constant
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    21
from rql.utils import rqlvar_maker
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    22
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    23
from cubicweb import dbapi, server
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    24
from cubicweb import BadConnectionId, UnknownEid, ConnectionError
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    25
from cubicweb.cwconfig import register_persistent_options
2625
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2593
diff changeset
    26
from cubicweb.server.sources import (AbstractSource, ConnectionWrapper,
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2593
diff changeset
    27
                                     TimedCache, dbg_st_search, dbg_results)
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    28
3016
5787d1cc8106 [rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2763
diff changeset
    29
5787d1cc8106 [rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2763
diff changeset
    30
def uidtype(union, col, etype, args):
5787d1cc8106 [rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2763
diff changeset
    31
    select, col = union.locate_subquery(col, etype, args)
5787d1cc8106 [rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2763
diff changeset
    32
    return getattr(select.selection[col], 'uidtype', None)
5787d1cc8106 [rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2763
diff changeset
    33
5787d1cc8106 [rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2763
diff changeset
    34
1138
22f634977c95 make pylint happy, fix some bugs on the way
sylvain.thenault@logilab.fr
parents: 1016
diff changeset
    35
class ReplaceByInOperator(Exception):
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    36
    def __init__(self, eids):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    37
        self.eids = eids
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
    38
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    39
class PyroRQLSource(AbstractSource):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    40
    """External repository source, using Pyro connection"""
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
    41
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    42
    # boolean telling if modification hooks should be called when something is
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    43
    # modified in this source
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    44
    should_call_hooks = False
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    45
    # boolean telling if the repository should connect to this source during
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    46
    # migration
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    47
    connect_for_migration = False
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
    48
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    49
    support_entities = None
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
    50
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    51
    options = (
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    52
        # XXX pyro-ns host/port
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    53
        ('pyro-ns-id',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    54
         {'type' : 'string',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    55
          'default': REQUIRED,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    56
          'help': 'identifier of the repository in the pyro name server',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    57
          'group': 'pyro-source', 'inputlevel': 0,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    58
          }),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    59
        ('mapping-file',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    60
         {'type' : 'string',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    61
          'default': REQUIRED,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    62
          'help': 'path to a python file with the schema mapping definition',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    63
          'group': 'pyro-source', 'inputlevel': 1,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    64
          }),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    65
        ('cubicweb-user',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    66
         {'type' : 'string',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    67
          'default': REQUIRED,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    68
          'help': 'user to use for connection on the distant repository',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    69
          'group': 'pyro-source', 'inputlevel': 0,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    70
          }),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    71
        ('cubicweb-password',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    72
         {'type' : 'password',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    73
          'default': '',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    74
          'help': 'user to use for connection on the distant repository',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    75
          'group': 'pyro-source', 'inputlevel': 0,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    76
          }),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    77
        ('base-url',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    78
         {'type' : 'string',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    79
          'default': '',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    80
          'help': 'url of the web site for the distant repository, if you want '
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    81
          'to generate external link to entities from this repository',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    82
          'group': 'pyro-source', 'inputlevel': 1,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    83
          }),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    84
        ('pyro-ns-host',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    85
         {'type' : 'string',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    86
          'default': None,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    87
          'help': 'Pyro name server\'s host. If not set, default to the value \
2665
0c6281487f90 [pyro] use lgc.pyro_ext, simplify pyro related options
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2625
diff changeset
    88
from all_in_one.conf. It may contains port information using <host>:<port> notation.',
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    89
          'group': 'pyro-source', 'inputlevel': 1,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    90
          }),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    91
        ('pyro-ns-group',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    92
         {'type' : 'string',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    93
          'default': None,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    94
          'help': 'Pyro name server\'s group where the repository will be \
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    95
registered. If not set, default to the value from all_in_one.conf.',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    96
          'group': 'pyro-source', 'inputlevel': 1,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    97
          }),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    98
        ('synchronization-interval',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    99
         {'type' : 'int',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   100
          'default': 5*60,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   101
          'help': 'interval between synchronization with the external \
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   102
repository (default to 5 minutes).',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   103
          'group': 'pyro-source', 'inputlevel': 2,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   104
          }),
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   105
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   106
    )
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   107
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   108
    PUBLIC_KEYS = AbstractSource.PUBLIC_KEYS + ('base-url',)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   109
    _conn = None
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   110
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   111
    def __init__(self, repo, appschema, source_config, *args, **kwargs):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   112
        AbstractSource.__init__(self, repo, appschema, source_config,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   113
                                *args, **kwargs)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   114
        mappingfile = source_config['mapping-file']
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   115
        if not mappingfile[0] == '/':
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   116
            mappingfile = join(repo.config.apphome, mappingfile)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   117
        mapping = {}
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   118
        execfile(mappingfile, mapping)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   119
        self.support_entities = mapping['support_entities']
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   120
        self.support_relations = mapping.get('support_relations', {})
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   121
        self.dont_cross_relations = mapping.get('dont_cross_relations', ())
417
828424d2eb3c oops, get back cross_relation from mapping to rql source
sylvain.thenault@logilab.fr
parents: 391
diff changeset
   122
        self.cross_relations = mapping.get('cross_relations', ())
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   123
        baseurl = source_config.get('base-url')
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   124
        if baseurl and not baseurl.endswith('/'):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   125
            source_config['base-url'] += '/'
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   126
        self.config = source_config
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   127
        myoptions = (('%s.latest-update-time' % self.uri,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   128
                      {'type' : 'int', 'sitewide': True,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   129
                       'default': 0,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   130
                       'help': _('timestamp of the latest source synchronization.'),
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   131
                       'group': 'sources',
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   132
                       }),)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   133
        register_persistent_options(myoptions)
1238
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   134
        self._query_cache = TimedCache(30)
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   135
2763
39b42e158249 [ms] proper reset cache on external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2665
diff changeset
   136
    def reset_caches(self):
39b42e158249 [ms] proper reset cache on external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2665
diff changeset
   137
        """method called during test to reset potential source caches"""
39b42e158249 [ms] proper reset cache on external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2665
diff changeset
   138
        self._query_cache = TimedCache(30)
39b42e158249 [ms] proper reset cache on external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2665
diff changeset
   139
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   140
    def last_update_time(self):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   141
        pkey = u'sources.%s.latest-update-time' % self.uri
1398
5fe84a5f7035 rename internal entity types to have CW prefix instead of E
sylvain.thenault@logilab.fr
parents: 1263
diff changeset
   142
        rql = 'Any V WHERE X is CWProperty, X value V, X pkey %(k)s'
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   143
        session = self.repo.internal_session()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   144
        try:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   145
            rset = session.execute(rql, {'k': pkey})
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   146
            if not rset:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   147
                # insert it
1398
5fe84a5f7035 rename internal entity types to have CW prefix instead of E
sylvain.thenault@logilab.fr
parents: 1263
diff changeset
   148
                session.execute('INSERT CWProperty X: X pkey %(k)s, X value %(v)s',
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   149
                                {'k': pkey, 'v': u'0'})
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   150
                session.commit()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   151
                timestamp = 0
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   152
            else:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   153
                assert len(rset) == 1
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   154
                timestamp = int(rset[0][0])
1016
26387b836099 use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   155
            return datetime.fromtimestamp(timestamp)
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   156
        finally:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   157
            session.close()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   158
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   159
    def init(self):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   160
        """method called by the repository once ready to handle request"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   161
        interval = int(self.config.get('synchronization-interval', 5*60))
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   162
        self.repo.looping_task(interval, self.synchronize)
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   163
        self.repo.looping_task(self._query_cache.ttl.seconds/10, self._query_cache.clear_expired)
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   164
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   165
    def synchronize(self, mtime=None):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   166
        """synchronize content known by this repository with content in the
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   167
        external repository
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   168
        """
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   169
        self.info('synchronizing pyro source %s', self.uri)
519
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   170
        cnx = self.get_connection()
2271
5483155a1e17 avoid additional tb in logs when we can't connect to the external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2269
diff changeset
   171
        try:
5483155a1e17 avoid additional tb in logs when we can't connect to the external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2269
diff changeset
   172
            extrepo = cnx._repo
5483155a1e17 avoid additional tb in logs when we can't connect to the external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2269
diff changeset
   173
        except AttributeError:
5483155a1e17 avoid additional tb in logs when we can't connect to the external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2269
diff changeset
   174
            # fake connection wrapper returned when we can't connect to the
5483155a1e17 avoid additional tb in logs when we can't connect to the external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2269
diff changeset
   175
            # external source (hence we've no chance to synchronize...)
5483155a1e17 avoid additional tb in logs when we can't connect to the external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2269
diff changeset
   176
            return
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   177
        etypes = self.support_entities.keys()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   178
        if mtime is None:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   179
            mtime = self.last_update_time()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   180
        updatetime, modified, deleted = extrepo.entities_modified_since(etypes,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   181
                                                                        mtime)
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   182
        self._query_cache.clear()
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   183
        repo = self.repo
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   184
        session = repo.internal_session()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   185
        try:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   186
            for etype, extid in modified:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   187
                try:
519
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   188
                    exturi = cnx.describe(extid)[1]
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   189
                    if exturi == 'system' or not exturi in repo.sources_by_uri:
1952
8e19c813750d fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1805
diff changeset
   190
                        eid = self.extid2eid(str(extid), etype, session)
519
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   191
                        rset = session.eid_rset(eid, etype)
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   192
                        entity = rset.get_entity(0, 0)
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   193
                        entity.complete(entity.e_schema.indexable_attributes())
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   194
                        repo.index_entity(session, entity)
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   195
                except:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   196
                    self.exception('while updating %s with external id %s of source %s',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   197
                                   etype, extid, self.uri)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   198
                    continue
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   199
            for etype, extid in deleted:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   200
                try:
1952
8e19c813750d fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1805
diff changeset
   201
                    eid = self.extid2eid(str(extid), etype, session,
8e19c813750d fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1805
diff changeset
   202
                                         insert=False)
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   203
                    # entity has been deleted from external repository but is not known here
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   204
                    if eid is not None:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   205
                        repo.delete_info(session, eid)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   206
                except:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   207
                    self.exception('while updating %s with external id %s of source %s',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   208
                                   etype, extid, self.uri)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   209
                    continue
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   210
            session.execute('SET X value %(v)s WHERE X pkey %(k)s',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   211
                            {'k': u'sources.%s.latest-update-time' % self.uri,
1016
26387b836099 use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   212
                             'v': unicode(int(mktime(updatetime.timetuple())))})
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   213
            session.commit()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   214
        finally:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   215
            session.close()
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   216
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   217
    def _get_connection(self):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   218
        """open and return a connection to the source"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   219
        nshost = self.config.get('pyro-ns-host') or self.repo.config['pyro-ns-host']
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   220
        nsgroup = self.config.get('pyro-ns-group') or self.repo.config['pyro-ns-group']
3527
efeb16ff93f3 log when connecting to an external pyro source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3043
diff changeset
   221
        self.info('connecting to instance :%s.%s for user %s',
3605
1466323ddb9c fix debug msg
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3527
diff changeset
   222
                  nsgroup, self.config['pyro-ns-id'], self.config['cubicweb-user'])
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   223
        #cnxprops = ConnectionProperties(cnxtype=self.config['cnx-type'])
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   224
        return dbapi.connect(database=self.config['pyro-ns-id'],
2269
0d1b7206fdc9 renamed arg
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1977
diff changeset
   225
                             login=self.config['cubicweb-user'],
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   226
                             password=self.config['cubicweb-password'],
2665
0c6281487f90 [pyro] use lgc.pyro_ext, simplify pyro related options
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2625
diff changeset
   227
                             host=nshost, group=nsgroup,
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   228
                             setvreg=False) #cnxprops=cnxprops)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   229
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   230
    def get_connection(self):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   231
        try:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   232
            return self._get_connection()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   233
        except (ConnectionError, PyroError):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   234
            self.critical("can't get connection to source %s", self.uri,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   235
                          exc_info=1)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   236
            return ConnectionWrapper()
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   237
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   238
    def check_connection(self, cnx):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   239
        """check connection validity, return None if the connection is still valid
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   240
        else a new connection
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   241
        """
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   242
        # we have to transfer manually thread ownership. This can be done safely
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   243
        # since the pool to which belong the connection is affected to one
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   244
        # session/thread and can't be called simultaneously
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   245
        try:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   246
            cnx._repo._transferThread(threading.currentThread())
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   247
        except AttributeError:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   248
            # inmemory connection
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   249
            pass
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   250
        if not isinstance(cnx, ConnectionWrapper):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   251
            try:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   252
                cnx.check()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   253
                return # ok
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   254
            except (BadConnectionId, ConnectionClosedError):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   255
                pass
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   256
        # try to reconnect
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   257
        return self.get_connection()
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   258
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   259
    def syntax_tree_search(self, session, union, args=None, cachekey=None,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   260
                           varmap=None):
2625
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2593
diff changeset
   261
        assert dbg_st_search(self.uri, union, varmap, args, cachekey)
1238
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   262
        rqlkey = union.as_string(kwargs=args)
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   263
        try:
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   264
            results = self._query_cache[rqlkey]
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   265
        except KeyError:
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   266
            results = self._syntax_tree_search(session, union, args)
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   267
            self._query_cache[rqlkey] = results
2625
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2593
diff changeset
   268
        assert dbg_results(results)
1238
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   269
        return results
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   270
1238
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   271
    def _syntax_tree_search(self, session, union, args):
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   272
        """return result from this source for a rql query (actually from a rql
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   273
        syntax tree and a solution dictionary mapping each used variable to a
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   274
        possible type). If cachekey is given, the query necessary to fetch the
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   275
        results (but not the results themselves) may be cached using this key.
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   276
        """
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   277
        if not args is None:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   278
            args = args.copy()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   279
        # get cached cursor anyway
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   280
        cu = session.pool[self.uri]
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   281
        if cu is None:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   282
            # this is a ConnectionWrapper instance
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   283
            msg = session._("can't connect to source %s, some data may be missing")
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   284
            session.set_shared_data('sources_error', msg % self.uri)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   285
            return []
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   286
        try:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   287
            rql, cachekey = RQL2RQL(self).generate(session, union, args)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   288
        except UnknownEid, ex:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   289
            if server.DEBUG:
2625
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2593
diff changeset
   290
                print '  unknown eid', ex, 'no results'
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   291
            return []
2593
16d9419a4a79 F: start to handle binary debug log level on the server side
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2271
diff changeset
   292
        if server.DEBUG & server.DBG_RQL:
2625
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2593
diff changeset
   293
            print '  translated rql', rql
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   294
        try:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   295
            rset = cu.execute(rql, args, cachekey)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   296
        except Exception, ex:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   297
            self.exception(str(ex))
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   298
            msg = session._("error while querying source %s, some data may be missing")
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   299
            session.set_shared_data('sources_error', msg % self.uri)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   300
            return []
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   301
        descr = rset.description
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   302
        if rset:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   303
            needtranslation = []
519
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   304
            rows = rset.rows
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   305
            for i, etype in enumerate(descr[0]):
3689
deb13e88e037 follow yams 0.25 api changes to improve performance
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3605
diff changeset
   306
                if (etype is None or not self.schema.eschema(etype).final
3016
5787d1cc8106 [rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2763
diff changeset
   307
                    or uidtype(union, i, etype, args)):
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   308
                    needtranslation.append(i)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   309
            if needtranslation:
519
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   310
                cnx = session.pool.connection(self.uri)
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   311
                for rowindex in xrange(rset.rowcount - 1, -1, -1):
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   312
                    row = rows[rowindex]
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   313
                    for colindex in needtranslation:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   314
                        if row[colindex] is not None: # optional variable
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   315
                            etype = descr[rowindex][colindex]
519
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   316
                            exttype, exturi, extid = cnx.describe(row[colindex])
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   317
                            if exturi == 'system' or not exturi in self.repo.sources_by_uri:
1952
8e19c813750d fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1805
diff changeset
   318
                                eid = self.extid2eid(str(row[colindex]), etype,
8e19c813750d fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1805
diff changeset
   319
                                                     session)
519
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   320
                                row[colindex] = eid
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   321
                            else:
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   322
                                # skip this row
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   323
                                del rows[rowindex]
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   324
                                del descr[rowindex]
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   325
                                break
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   326
            results = rows
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   327
        else:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   328
            results = []
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   329
        return results
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   330
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   331
    def _entity_relations_and_kwargs(self, session, entity):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   332
        relations = []
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   333
        kwargs = {'x': self.eid2extid(entity.eid, session)}
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   334
        for key, val in entity.iteritems():
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   335
            relations.append('X %s %%(%s)s' % (key, key))
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   336
            kwargs[key] = val
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   337
        return relations, kwargs
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   338
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   339
    def add_entity(self, session, entity):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   340
        """add a new entity to the source"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   341
        raise NotImplementedError()
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   342
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   343
    def update_entity(self, session, entity):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   344
        """update an entity in the source"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   345
        relations, kwargs = self._entity_relations_and_kwargs(session, entity)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   346
        cu = session.pool[self.uri]
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   347
        cu.execute('SET %s WHERE X eid %%(x)s' % ','.join(relations),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   348
                   kwargs, 'x')
1805
7bad2785d167 clear query cache on write
sylvain.thenault@logilab.fr
parents: 1791
diff changeset
   349
        self._query_cache.clear()
3043
7bcb7f1c0f46 force cache clear when external entity is edited, we don't know what may occurs in external repository hooks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3016
diff changeset
   350
        entity.clear_all_caches()
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   351
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   352
    def delete_entity(self, session, etype, eid):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   353
        """delete an entity from the source"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   354
        cu = session.pool[self.uri]
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   355
        cu.execute('DELETE %s X WHERE X eid %%(x)s' % etype,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   356
                   {'x': self.eid2extid(eid, session)}, 'x')
1805
7bad2785d167 clear query cache on write
sylvain.thenault@logilab.fr
parents: 1791
diff changeset
   357
        self._query_cache.clear()
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   358
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   359
    def add_relation(self, session, subject, rtype, object):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   360
        """add a relation to the source"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   361
        cu = session.pool[self.uri]
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   362
        cu.execute('SET X %s Y WHERE X eid %%(x)s, Y eid %%(y)s' % rtype,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   363
                   {'x': self.eid2extid(subject, session),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   364
                    'y': self.eid2extid(object, session)}, ('x', 'y'))
1805
7bad2785d167 clear query cache on write
sylvain.thenault@logilab.fr
parents: 1791
diff changeset
   365
        self._query_cache.clear()
3043
7bcb7f1c0f46 force cache clear when external entity is edited, we don't know what may occurs in external repository hooks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3016
diff changeset
   366
        session.entity_from_eid(subject).clear_all_caches()
7bcb7f1c0f46 force cache clear when external entity is edited, we don't know what may occurs in external repository hooks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3016
diff changeset
   367
        session.entity_from_eid(object).clear_all_caches()
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   368
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   369
    def delete_relation(self, session, subject, rtype, object):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   370
        """delete a relation from the source"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   371
        cu = session.pool[self.uri]
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   372
        cu.execute('DELETE X %s Y WHERE X eid %%(x)s, Y eid %%(y)s' % rtype,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   373
                   {'x': self.eid2extid(subject, session),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   374
                    'y': self.eid2extid(object, session)}, ('x', 'y'))
1805
7bad2785d167 clear query cache on write
sylvain.thenault@logilab.fr
parents: 1791
diff changeset
   375
        self._query_cache.clear()
3043
7bcb7f1c0f46 force cache clear when external entity is edited, we don't know what may occurs in external repository hooks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3016
diff changeset
   376
        session.entity_from_eid(subject).clear_all_caches()
7bcb7f1c0f46 force cache clear when external entity is edited, we don't know what may occurs in external repository hooks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3016
diff changeset
   377
        session.entity_from_eid(object).clear_all_caches()
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   378
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   379
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   380
class RQL2RQL(object):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   381
    """translate a local rql query to be executed on a distant repository"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   382
    def __init__(self, source):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   383
        self.source = source
390
739d12586b9d avoid potentialy mis-interpreted attribute error
sylvain.thenault@logilab.fr
parents: 257
diff changeset
   384
        self.current_operator = None
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   385
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   386
    def _accept_children(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   387
        res = []
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   388
        for child in node.children:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   389
            rql = child.accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   390
            if rql is not None:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   391
                res.append(rql)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   392
        return res
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   393
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   394
    def generate(self, session, rqlst, args):
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   395
        self._session = session
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   396
        self.kwargs = args
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   397
        self.cachekey = []
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   398
        self.need_translation = False
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   399
        return self.visit_union(rqlst), self.cachekey
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   400
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   401
    def visit_union(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   402
        s = self._accept_children(node)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   403
        if len(s) > 1:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   404
            return ' UNION '.join('(%s)' % q for q in s)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   405
        return s[0]
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   406
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   407
    def visit_select(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   408
        """return the tree as an encoded rql string"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   409
        self._varmaker = rqlvar_maker(defined=node.defined_vars.copy())
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   410
        self._const_var = {}
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   411
        if node.distinct:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   412
            base = 'DISTINCT Any'
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   413
        else:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   414
            base = 'Any'
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   415
        s = ['%s %s' % (base, ','.join(v.accept(self) for v in node.selection))]
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   416
        if node.groupby:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   417
            s.append('GROUPBY %s' % ', '.join(group.accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   418
                                              for group in node.groupby))
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   419
        if node.orderby:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   420
            s.append('ORDERBY %s' % ', '.join(self.visit_sortterm(term)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   421
                                              for term in node.orderby))
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   422
        if node.limit is not None:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   423
            s.append('LIMIT %s' % node.limit)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   424
        if node.offset:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   425
            s.append('OFFSET %s' % node.offset)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   426
        restrictions = []
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   427
        if node.where is not None:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   428
            nr = node.where.accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   429
            if nr is not None:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   430
                restrictions.append(nr)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   431
        if restrictions:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   432
            s.append('WHERE %s' % ','.join(restrictions))
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   433
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   434
        if node.having:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   435
            s.append('HAVING %s' % ', '.join(term.accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   436
                                             for term in node.having))
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   437
        subqueries = []
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   438
        for subquery in node.with_:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   439
            subqueries.append('%s BEING (%s)' % (','.join(ca.name for ca in subquery.aliases),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   440
                                                 self.visit_union(subquery.query)))
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   441
        if subqueries:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   442
            s.append('WITH %s' % (','.join(subqueries)))
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   443
        return ' '.join(s)
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   444
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   445
    def visit_and(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   446
        res = self._accept_children(node)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   447
        if res:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   448
            return ', '.join(res)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   449
        return
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   450
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   451
    def visit_or(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   452
        res = self._accept_children(node)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   453
        if len(res) > 1:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   454
            return ' OR '.join('(%s)' % rql for rql in res)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   455
        elif res:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   456
            return res[0]
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   457
        return
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   458
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   459
    def visit_not(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   460
        rql = node.children[0].accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   461
        if rql:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   462
            return 'NOT (%s)' % rql
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   463
        return
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   464
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   465
    def visit_exists(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   466
        return 'EXISTS(%s)' % node.children[0].accept(self)
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   467
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   468
    def visit_relation(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   469
        try:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   470
            if isinstance(node.children[0], Constant):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   471
                # simplified rqlst, reintroduce eid relation
391
42693fe3ef6f skip neged relation here as well
sylvain.thenault@logilab.fr
parents: 390
diff changeset
   472
                try:
42693fe3ef6f skip neged relation here as well
sylvain.thenault@logilab.fr
parents: 390
diff changeset
   473
                    restr, lhs = self.process_eid_const(node.children[0])
42693fe3ef6f skip neged relation here as well
sylvain.thenault@logilab.fr
parents: 390
diff changeset
   474
                except UnknownEid:
42693fe3ef6f skip neged relation here as well
sylvain.thenault@logilab.fr
parents: 390
diff changeset
   475
                    # can safely skip not relation with an unsupported eid
42693fe3ef6f skip neged relation here as well
sylvain.thenault@logilab.fr
parents: 390
diff changeset
   476
                    if node.neged(strict=True):
42693fe3ef6f skip neged relation here as well
sylvain.thenault@logilab.fr
parents: 390
diff changeset
   477
                        return
42693fe3ef6f skip neged relation here as well
sylvain.thenault@logilab.fr
parents: 390
diff changeset
   478
                    raise
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   479
            else:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   480
                lhs = node.children[0].accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   481
                restr = None
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   482
        except UnknownEid:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   483
            # can safely skip not relation with an unsupported eid
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   484
            if node.neged(strict=True):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   485
                return
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   486
            # XXX what about optional relation or outer NOT EXISTS()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   487
            raise
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   488
        if node.optional in ('left', 'both'):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   489
            lhs += '?'
3689
deb13e88e037 follow yams 0.25 api changes to improve performance
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3605
diff changeset
   490
        if node.r_type == 'eid' or not self.source.schema.rschema(node.r_type).final:
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   491
            self.need_translation = True
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   492
            self.current_operator = node.operator()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   493
            if isinstance(node.children[0], Constant):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   494
                self.current_etypes = (node.children[0].uidtype,)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   495
            else:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   496
                self.current_etypes = node.children[0].variable.stinfo['possibletypes']
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   497
        try:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   498
            rhs = node.children[1].accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   499
        except UnknownEid:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   500
            # can safely skip not relation with an unsupported eid
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   501
            if node.neged(strict=True):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   502
                return
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   503
            # XXX what about optional relation or outer NOT EXISTS()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   504
            raise
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   505
        except ReplaceByInOperator, ex:
1952
8e19c813750d fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1805
diff changeset
   506
            rhs = 'IN (%s)' % ','.join(eid for eid in ex.eids)
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   507
        self.need_translation = False
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   508
        self.current_operator = None
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   509
        if node.optional in ('right', 'both'):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   510
            rhs += '?'
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   511
        if restr is not None:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   512
            return '%s %s %s, %s' % (lhs, node.r_type, rhs, restr)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   513
        return '%s %s %s' % (lhs, node.r_type, rhs)
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   514
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   515
    def visit_comparison(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   516
        if node.operator in ('=', 'IS'):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   517
            return node.children[0].accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   518
        return '%s %s' % (node.operator.encode(),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   519
                          node.children[0].accept(self))
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   520
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   521
    def visit_mathexpression(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   522
        return '(%s %s %s)' % (node.children[0].accept(self),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   523
                               node.operator.encode(),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   524
                               node.children[1].accept(self))
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   525
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   526
    def visit_function(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   527
        #if node.name == 'IN':
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   528
        res = []
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   529
        for child in node.children:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   530
            try:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   531
                rql = child.accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   532
            except UnknownEid, ex:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   533
                continue
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   534
            res.append(rql)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   535
        if not res:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   536
            raise ex
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   537
        return '%s(%s)' % (node.name, ', '.join(res))
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   538
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   539
    def visit_constant(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   540
        if self.need_translation or node.uidtype:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   541
            if node.type == 'Int':
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   542
                return str(self.eid2extid(node.value))
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   543
            if node.type == 'Substitute':
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   544
                key = node.value
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   545
                # ensure we have not yet translated the value...
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   546
                if not key in self._const_var:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   547
                    self.kwargs[key] = self.eid2extid(self.kwargs[key])
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   548
                    self.cachekey.append(key)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   549
                    self._const_var[key] = None
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   550
        return node.as_string()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   551
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   552
    def visit_variableref(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   553
        """get the sql name for a variable reference"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   554
        return node.name
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   555
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   556
    def visit_sortterm(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   557
        if node.asc:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   558
            return node.term.accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   559
        return '%s DESC' % node.term.accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   560
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   561
    def process_eid_const(self, const):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   562
        value = const.eval(self.kwargs)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   563
        try:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   564
            return None, self._const_var[value]
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   565
        except:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   566
            var = self._varmaker.next()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   567
            self.need_translation = True
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   568
            restr = '%s eid %s' % (var, self.visit_constant(const))
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   569
            self.need_translation = False
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   570
            self._const_var[value] = var
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   571
            return restr, var
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   572
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   573
    def eid2extid(self, eid):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   574
        try:
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   575
            return self.source.eid2extid(eid, self._session)
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   576
        except UnknownEid:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   577
            operator = self.current_operator
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   578
            if operator is not None and operator != '=':
1954
9b20f3504af8 cleanup
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1952
diff changeset
   579
                # deal with query like "X eid > 12"
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   580
                #
1954
9b20f3504af8 cleanup
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1952
diff changeset
   581
                # The problem is that eid order in the external source may
9b20f3504af8 cleanup
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1952
diff changeset
   582
                # differ from the local source
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   583
                #
1954
9b20f3504af8 cleanup
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1952
diff changeset
   584
                # So search for all eids from this source matching the condition
9b20f3504af8 cleanup
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1952
diff changeset
   585
                # locally and then to replace the "> 12" branch by "IN (eids)"
9b20f3504af8 cleanup
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1952
diff changeset
   586
                #
9b20f3504af8 cleanup
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1952
diff changeset
   587
                # XXX we may have to insert a huge number of eids...)
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   588
                sql = "SELECT extid FROM entities WHERE source='%s' AND type IN (%s) AND eid%s%s"
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   589
                etypes = ','.join("'%s'" % etype for etype in self.current_etypes)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   590
                cu = self._session.system_sql(sql % (self.source.uri, etypes,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   591
                                                      operator, eid))
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   592
                # XXX buggy cu.rowcount which may be zero while there are some
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   593
                # results
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   594
                rows = cu.fetchall()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   595
                if rows:
1952
8e19c813750d fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1805
diff changeset
   596
                    raise ReplaceByInOperator((b64decode(r[0]) for r in rows))
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   597
            raise
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   598