server/sources/pyrorql.py
author Sylvain Thénault <sylvain.thenault@logilab.fr>
Tue, 20 Apr 2010 18:34:07 +0200
changeset 5363 d336476dcf57
parent 5323 329b4f6d18b4
child 5423 e15abfdcce38
permissions -rw-r--r--
[3.8 api update] remaining cache keys
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     1
"""Source to query another RQL repository using pyro
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     2
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     3
:organization: Logilab
4212
ab6573088b4a update copyright: welcome 2010
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents: 3689
diff changeset
     4
:copyright: 2007-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2.
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     5
:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr
1977
606923dff11b big bunch of copyright / docstring update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents: 1954
diff changeset
     6
:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     7
"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     8
__docformat__ = "restructuredtext en"
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     9
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    10
import threading
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    11
from os.path import join
1016
26387b836099 use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents: 520
diff changeset
    12
from time import mktime
26387b836099 use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents: 520
diff changeset
    13
from datetime import datetime
1952
8e19c813750d fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1805
diff changeset
    14
from base64 import b64decode
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    15
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    16
from Pyro.errors import PyroError, ConnectionClosedError
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    17
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    18
from logilab.common.configuration import REQUIRED
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    19
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    20
from rql.nodes import Constant
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    21
from rql.utils import rqlvar_maker
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    22
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    23
from cubicweb import dbapi, server
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    24
from cubicweb import BadConnectionId, UnknownEid, ConnectionError
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    25
from cubicweb.cwconfig import register_persistent_options
2625
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2593
diff changeset
    26
from cubicweb.server.sources import (AbstractSource, ConnectionWrapper,
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2593
diff changeset
    27
                                     TimedCache, dbg_st_search, dbg_results)
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    28
3016
5787d1cc8106 [rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2763
diff changeset
    29
5787d1cc8106 [rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2763
diff changeset
    30
def uidtype(union, col, etype, args):
5787d1cc8106 [rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2763
diff changeset
    31
    select, col = union.locate_subquery(col, etype, args)
5787d1cc8106 [rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2763
diff changeset
    32
    return getattr(select.selection[col], 'uidtype', None)
5787d1cc8106 [rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2763
diff changeset
    33
5787d1cc8106 [rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2763
diff changeset
    34
1138
22f634977c95 make pylint happy, fix some bugs on the way
sylvain.thenault@logilab.fr
parents: 1016
diff changeset
    35
class ReplaceByInOperator(Exception):
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    36
    def __init__(self, eids):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    37
        self.eids = eids
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
    38
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    39
class PyroRQLSource(AbstractSource):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    40
    """External repository source, using Pyro connection"""
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
    41
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    42
    # boolean telling if modification hooks should be called when something is
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    43
    # modified in this source
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    44
    should_call_hooks = False
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    45
    # boolean telling if the repository should connect to this source during
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    46
    # migration
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    47
    connect_for_migration = False
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
    48
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    49
    support_entities = None
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
    50
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    51
    options = (
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    52
        # XXX pyro-ns host/port
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    53
        ('pyro-ns-id',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    54
         {'type' : 'string',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    55
          'default': REQUIRED,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    56
          'help': 'identifier of the repository in the pyro name server',
5323
329b4f6d18b4 [config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5174
diff changeset
    57
          'group': 'pyro-source', 'level': 0,
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    58
          }),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    59
        ('mapping-file',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    60
         {'type' : 'string',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    61
          'default': REQUIRED,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    62
          'help': 'path to a python file with the schema mapping definition',
5323
329b4f6d18b4 [config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5174
diff changeset
    63
          'group': 'pyro-source', 'level': 1,
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    64
          }),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    65
        ('cubicweb-user',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    66
         {'type' : 'string',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    67
          'default': REQUIRED,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    68
          'help': 'user to use for connection on the distant repository',
5323
329b4f6d18b4 [config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5174
diff changeset
    69
          'group': 'pyro-source', 'level': 0,
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    70
          }),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    71
        ('cubicweb-password',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    72
         {'type' : 'password',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    73
          'default': '',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    74
          'help': 'user to use for connection on the distant repository',
5323
329b4f6d18b4 [config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5174
diff changeset
    75
          'group': 'pyro-source', 'level': 0,
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    76
          }),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    77
        ('base-url',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    78
         {'type' : 'string',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    79
          'default': '',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    80
          'help': 'url of the web site for the distant repository, if you want '
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    81
          'to generate external link to entities from this repository',
5323
329b4f6d18b4 [config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5174
diff changeset
    82
          'group': 'pyro-source', 'level': 1,
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    83
          }),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    84
        ('pyro-ns-host',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    85
         {'type' : 'string',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    86
          'default': None,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    87
          'help': 'Pyro name server\'s host. If not set, default to the value \
2665
0c6281487f90 [pyro] use lgc.pyro_ext, simplify pyro related options
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2625
diff changeset
    88
from all_in_one.conf. It may contains port information using <host>:<port> notation.',
5323
329b4f6d18b4 [config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5174
diff changeset
    89
          'group': 'pyro-source', 'level': 1,
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    90
          }),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    91
        ('pyro-ns-group',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    92
         {'type' : 'string',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    93
          'default': None,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    94
          'help': 'Pyro name server\'s group where the repository will be \
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    95
registered. If not set, default to the value from all_in_one.conf.',
5323
329b4f6d18b4 [config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5174
diff changeset
    96
          'group': 'pyro-source', 'level': 1,
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    97
          }),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    98
        ('synchronization-interval',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    99
         {'type' : 'int',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   100
          'default': 5*60,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   101
          'help': 'interval between synchronization with the external \
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   102
repository (default to 5 minutes).',
5323
329b4f6d18b4 [config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5174
diff changeset
   103
          'group': 'pyro-source', 'level': 2,
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   104
          }),
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   105
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   106
    )
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   107
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   108
    PUBLIC_KEYS = AbstractSource.PUBLIC_KEYS + ('base-url',)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   109
    _conn = None
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   110
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   111
    def __init__(self, repo, appschema, source_config, *args, **kwargs):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   112
        AbstractSource.__init__(self, repo, appschema, source_config,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   113
                                *args, **kwargs)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   114
        mappingfile = source_config['mapping-file']
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   115
        if not mappingfile[0] == '/':
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   116
            mappingfile = join(repo.config.apphome, mappingfile)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   117
        mapping = {}
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   118
        execfile(mappingfile, mapping)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   119
        self.support_entities = mapping['support_entities']
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   120
        self.support_relations = mapping.get('support_relations', {})
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   121
        self.dont_cross_relations = mapping.get('dont_cross_relations', ())
417
828424d2eb3c oops, get back cross_relation from mapping to rql source
sylvain.thenault@logilab.fr
parents: 391
diff changeset
   122
        self.cross_relations = mapping.get('cross_relations', ())
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   123
        baseurl = source_config.get('base-url')
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   124
        if baseurl and not baseurl.endswith('/'):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   125
            source_config['base-url'] += '/'
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   126
        self.config = source_config
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   127
        myoptions = (('%s.latest-update-time' % self.uri,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   128
                      {'type' : 'int', 'sitewide': True,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   129
                       'default': 0,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   130
                       'help': _('timestamp of the latest source synchronization.'),
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   131
                       'group': 'sources',
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   132
                       }),)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   133
        register_persistent_options(myoptions)
1238
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   134
        self._query_cache = TimedCache(30)
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   135
2763
39b42e158249 [ms] proper reset cache on external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2665
diff changeset
   136
    def reset_caches(self):
39b42e158249 [ms] proper reset cache on external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2665
diff changeset
   137
        """method called during test to reset potential source caches"""
39b42e158249 [ms] proper reset cache on external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2665
diff changeset
   138
        self._query_cache = TimedCache(30)
39b42e158249 [ms] proper reset cache on external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2665
diff changeset
   139
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   140
    def last_update_time(self):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   141
        pkey = u'sources.%s.latest-update-time' % self.uri
1398
5fe84a5f7035 rename internal entity types to have CW prefix instead of E
sylvain.thenault@logilab.fr
parents: 1263
diff changeset
   142
        rql = 'Any V WHERE X is CWProperty, X value V, X pkey %(k)s'
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   143
        session = self.repo.internal_session()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   144
        try:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   145
            rset = session.execute(rql, {'k': pkey})
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   146
            if not rset:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   147
                # insert it
1398
5fe84a5f7035 rename internal entity types to have CW prefix instead of E
sylvain.thenault@logilab.fr
parents: 1263
diff changeset
   148
                session.execute('INSERT CWProperty X: X pkey %(k)s, X value %(v)s',
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   149
                                {'k': pkey, 'v': u'0'})
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   150
                session.commit()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   151
                timestamp = 0
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   152
            else:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   153
                assert len(rset) == 1
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   154
                timestamp = int(rset[0][0])
1016
26387b836099 use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   155
            return datetime.fromtimestamp(timestamp)
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   156
        finally:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   157
            session.close()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   158
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   159
    def init(self):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   160
        """method called by the repository once ready to handle request"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   161
        interval = int(self.config.get('synchronization-interval', 5*60))
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   162
        self.repo.looping_task(interval, self.synchronize)
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   163
        self.repo.looping_task(self._query_cache.ttl.seconds/10, self._query_cache.clear_expired)
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   164
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   165
    def synchronize(self, mtime=None):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   166
        """synchronize content known by this repository with content in the
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   167
        external repository
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   168
        """
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   169
        self.info('synchronizing pyro source %s', self.uri)
519
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   170
        cnx = self.get_connection()
2271
5483155a1e17 avoid additional tb in logs when we can't connect to the external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2269
diff changeset
   171
        try:
5483155a1e17 avoid additional tb in logs when we can't connect to the external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2269
diff changeset
   172
            extrepo = cnx._repo
5483155a1e17 avoid additional tb in logs when we can't connect to the external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2269
diff changeset
   173
        except AttributeError:
5483155a1e17 avoid additional tb in logs when we can't connect to the external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2269
diff changeset
   174
            # fake connection wrapper returned when we can't connect to the
5483155a1e17 avoid additional tb in logs when we can't connect to the external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2269
diff changeset
   175
            # external source (hence we've no chance to synchronize...)
5483155a1e17 avoid additional tb in logs when we can't connect to the external source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2269
diff changeset
   176
            return
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   177
        etypes = self.support_entities.keys()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   178
        if mtime is None:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   179
            mtime = self.last_update_time()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   180
        updatetime, modified, deleted = extrepo.entities_modified_since(etypes,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   181
                                                                        mtime)
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   182
        self._query_cache.clear()
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   183
        repo = self.repo
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   184
        session = repo.internal_session()
4816
c02583cb80a9 repair stuff broken by fti handling changes
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4212
diff changeset
   185
        source = repo.system_source
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   186
        try:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   187
            for etype, extid in modified:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   188
                try:
519
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   189
                    exturi = cnx.describe(extid)[1]
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   190
                    if exturi == 'system' or not exturi in repo.sources_by_uri:
1952
8e19c813750d fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1805
diff changeset
   191
                        eid = self.extid2eid(str(extid), etype, session)
519
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   192
                        rset = session.eid_rset(eid, etype)
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   193
                        entity = rset.get_entity(0, 0)
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   194
                        entity.complete(entity.e_schema.indexable_attributes())
4816
c02583cb80a9 repair stuff broken by fti handling changes
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4212
diff changeset
   195
                        source.index_entity(session, entity)
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   196
                except:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   197
                    self.exception('while updating %s with external id %s of source %s',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   198
                                   etype, extid, self.uri)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   199
                    continue
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   200
            for etype, extid in deleted:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   201
                try:
1952
8e19c813750d fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1805
diff changeset
   202
                    eid = self.extid2eid(str(extid), etype, session,
8e19c813750d fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1805
diff changeset
   203
                                         insert=False)
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   204
                    # entity has been deleted from external repository but is not known here
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   205
                    if eid is not None:
4913
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 4816
diff changeset
   206
                        entity = session.entity_from_eid(eid, etype)
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 4816
diff changeset
   207
                        repo.delete_info(session, entity, self.uri, extid)
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   208
                except:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   209
                    self.exception('while updating %s with external id %s of source %s',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   210
                                   etype, extid, self.uri)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   211
                    continue
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   212
            session.execute('SET X value %(v)s WHERE X pkey %(k)s',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   213
                            {'k': u'sources.%s.latest-update-time' % self.uri,
1016
26387b836099 use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   214
                             'v': unicode(int(mktime(updatetime.timetuple())))})
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   215
            session.commit()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   216
        finally:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   217
            session.close()
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   218
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   219
    def _get_connection(self):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   220
        """open and return a connection to the source"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   221
        nshost = self.config.get('pyro-ns-host') or self.repo.config['pyro-ns-host']
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   222
        nsgroup = self.config.get('pyro-ns-group') or self.repo.config['pyro-ns-group']
3527
efeb16ff93f3 log when connecting to an external pyro source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3043
diff changeset
   223
        self.info('connecting to instance :%s.%s for user %s',
3605
1466323ddb9c fix debug msg
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3527
diff changeset
   224
                  nsgroup, self.config['pyro-ns-id'], self.config['cubicweb-user'])
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   225
        #cnxprops = ConnectionProperties(cnxtype=self.config['cnx-type'])
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   226
        return dbapi.connect(database=self.config['pyro-ns-id'],
2269
0d1b7206fdc9 renamed arg
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1977
diff changeset
   227
                             login=self.config['cubicweb-user'],
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   228
                             password=self.config['cubicweb-password'],
2665
0c6281487f90 [pyro] use lgc.pyro_ext, simplify pyro related options
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2625
diff changeset
   229
                             host=nshost, group=nsgroup,
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   230
                             setvreg=False) #cnxprops=cnxprops)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   231
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   232
    def get_connection(self):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   233
        try:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   234
            return self._get_connection()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   235
        except (ConnectionError, PyroError):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   236
            self.critical("can't get connection to source %s", self.uri,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   237
                          exc_info=1)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   238
            return ConnectionWrapper()
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   239
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   240
    def check_connection(self, cnx):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   241
        """check connection validity, return None if the connection is still valid
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   242
        else a new connection
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   243
        """
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   244
        # we have to transfer manually thread ownership. This can be done safely
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   245
        # since the pool to which belong the connection is affected to one
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   246
        # session/thread and can't be called simultaneously
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   247
        try:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   248
            cnx._repo._transferThread(threading.currentThread())
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   249
        except AttributeError:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   250
            # inmemory connection
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   251
            pass
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   252
        if not isinstance(cnx, ConnectionWrapper):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   253
            try:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   254
                cnx.check()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   255
                return # ok
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   256
            except (BadConnectionId, ConnectionClosedError):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   257
                pass
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   258
        # try to reconnect
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   259
        return self.get_connection()
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   260
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   261
    def syntax_tree_search(self, session, union, args=None, cachekey=None,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   262
                           varmap=None):
2625
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2593
diff changeset
   263
        assert dbg_st_search(self.uri, union, varmap, args, cachekey)
1238
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   264
        rqlkey = union.as_string(kwargs=args)
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   265
        try:
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   266
            results = self._query_cache[rqlkey]
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   267
        except KeyError:
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   268
            results = self._syntax_tree_search(session, union, args)
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   269
            self._query_cache[rqlkey] = results
2625
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2593
diff changeset
   270
        assert dbg_results(results)
1238
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   271
        return results
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   272
1238
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   273
    def _syntax_tree_search(self, session, union, args):
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   274
        """return result from this source for a rql query (actually from a rql
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   275
        syntax tree and a solution dictionary mapping each used variable to a
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   276
        possible type). If cachekey is given, the query necessary to fetch the
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   277
        results (but not the results themselves) may be cached using this key.
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   278
        """
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   279
        if not args is None:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   280
            args = args.copy()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   281
        # get cached cursor anyway
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   282
        cu = session.pool[self.uri]
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   283
        if cu is None:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   284
            # this is a ConnectionWrapper instance
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   285
            msg = session._("can't connect to source %s, some data may be missing")
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   286
            session.set_shared_data('sources_error', msg % self.uri)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   287
            return []
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   288
        try:
5174
78438ad513ca #759035: Automate addition of eid cachekey in RQL analysis
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4913
diff changeset
   289
            rql = RQL2RQL(self).generate(session, union, args)
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   290
        except UnknownEid, ex:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   291
            if server.DEBUG:
2625
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2593
diff changeset
   292
                print '  unknown eid', ex, 'no results'
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   293
            return []
2593
16d9419a4a79 F: start to handle binary debug log level on the server side
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2271
diff changeset
   294
        if server.DEBUG & server.DBG_RQL:
2625
d6012db7b93e R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2593
diff changeset
   295
            print '  translated rql', rql
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   296
        try:
5174
78438ad513ca #759035: Automate addition of eid cachekey in RQL analysis
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4913
diff changeset
   297
            rset = cu.execute(rql, args)
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   298
        except Exception, ex:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   299
            self.exception(str(ex))
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   300
            msg = session._("error while querying source %s, some data may be missing")
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   301
            session.set_shared_data('sources_error', msg % self.uri)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   302
            return []
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   303
        descr = rset.description
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   304
        if rset:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   305
            needtranslation = []
519
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   306
            rows = rset.rows
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   307
            for i, etype in enumerate(descr[0]):
3689
deb13e88e037 follow yams 0.25 api changes to improve performance
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3605
diff changeset
   308
                if (etype is None or not self.schema.eschema(etype).final
3016
5787d1cc8106 [rset] fix #231354 w/ rql 0.22.3 api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 2763
diff changeset
   309
                    or uidtype(union, i, etype, args)):
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   310
                    needtranslation.append(i)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   311
            if needtranslation:
519
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   312
                cnx = session.pool.connection(self.uri)
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   313
                for rowindex in xrange(rset.rowcount - 1, -1, -1):
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   314
                    row = rows[rowindex]
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   315
                    for colindex in needtranslation:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   316
                        if row[colindex] is not None: # optional variable
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   317
                            etype = descr[rowindex][colindex]
519
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   318
                            exttype, exturi, extid = cnx.describe(row[colindex])
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   319
                            if exturi == 'system' or not exturi in self.repo.sources_by_uri:
1952
8e19c813750d fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1805
diff changeset
   320
                                eid = self.extid2eid(str(row[colindex]), etype,
8e19c813750d fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1805
diff changeset
   321
                                                     session)
519
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   322
                                row[colindex] = eid
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   323
                            else:
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   324
                                # skip this row
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   325
                                del rows[rowindex]
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   326
                                del descr[rowindex]
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   327
                                break
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   328
            results = rows
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   329
        else:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   330
            results = []
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   331
        return results
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   332
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   333
    def _entity_relations_and_kwargs(self, session, entity):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   334
        relations = []
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   335
        kwargs = {'x': self.eid2extid(entity.eid, session)}
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   336
        for key, val in entity.iteritems():
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   337
            relations.append('X %s %%(%s)s' % (key, key))
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   338
            kwargs[key] = val
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   339
        return relations, kwargs
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   340
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   341
    def add_entity(self, session, entity):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   342
        """add a new entity to the source"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   343
        raise NotImplementedError()
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   344
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   345
    def update_entity(self, session, entity):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   346
        """update an entity in the source"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   347
        relations, kwargs = self._entity_relations_and_kwargs(session, entity)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   348
        cu = session.pool[self.uri]
5363
d336476dcf57 [3.8 api update] remaining cache keys
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5323
diff changeset
   349
        cu.execute('SET %s WHERE X eid %%(x)s' % ','.join(relations), kwargs)
1805
7bad2785d167 clear query cache on write
sylvain.thenault@logilab.fr
parents: 1791
diff changeset
   350
        self._query_cache.clear()
3043
7bcb7f1c0f46 force cache clear when external entity is edited, we don't know what may occurs in external repository hooks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3016
diff changeset
   351
        entity.clear_all_caches()
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   352
4913
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 4816
diff changeset
   353
    def delete_entity(self, session, entity):
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   354
        """delete an entity from the source"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   355
        cu = session.pool[self.uri]
4913
083b4d454192 server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents: 4816
diff changeset
   356
        cu.execute('DELETE %s X WHERE X eid %%(x)s' % entity.__regid__,
5363
d336476dcf57 [3.8 api update] remaining cache keys
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5323
diff changeset
   357
                   {'x': self.eid2extid(entity.eid, session)})
1805
7bad2785d167 clear query cache on write
sylvain.thenault@logilab.fr
parents: 1791
diff changeset
   358
        self._query_cache.clear()
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   359
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   360
    def add_relation(self, session, subject, rtype, object):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   361
        """add a relation to the source"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   362
        cu = session.pool[self.uri]
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   363
        cu.execute('SET X %s Y WHERE X eid %%(x)s, Y eid %%(y)s' % rtype,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   364
                   {'x': self.eid2extid(subject, session),
5363
d336476dcf57 [3.8 api update] remaining cache keys
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5323
diff changeset
   365
                    'y': self.eid2extid(object, session)})
1805
7bad2785d167 clear query cache on write
sylvain.thenault@logilab.fr
parents: 1791
diff changeset
   366
        self._query_cache.clear()
3043
7bcb7f1c0f46 force cache clear when external entity is edited, we don't know what may occurs in external repository hooks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3016
diff changeset
   367
        session.entity_from_eid(subject).clear_all_caches()
7bcb7f1c0f46 force cache clear when external entity is edited, we don't know what may occurs in external repository hooks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3016
diff changeset
   368
        session.entity_from_eid(object).clear_all_caches()
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   369
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   370
    def delete_relation(self, session, subject, rtype, object):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   371
        """delete a relation from the source"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   372
        cu = session.pool[self.uri]
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   373
        cu.execute('DELETE X %s Y WHERE X eid %%(x)s, Y eid %%(y)s' % rtype,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   374
                   {'x': self.eid2extid(subject, session),
5363
d336476dcf57 [3.8 api update] remaining cache keys
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 5323
diff changeset
   375
                    'y': self.eid2extid(object, session)})
1805
7bad2785d167 clear query cache on write
sylvain.thenault@logilab.fr
parents: 1791
diff changeset
   376
        self._query_cache.clear()
3043
7bcb7f1c0f46 force cache clear when external entity is edited, we don't know what may occurs in external repository hooks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3016
diff changeset
   377
        session.entity_from_eid(subject).clear_all_caches()
7bcb7f1c0f46 force cache clear when external entity is edited, we don't know what may occurs in external repository hooks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3016
diff changeset
   378
        session.entity_from_eid(object).clear_all_caches()
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   379
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   380
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   381
class RQL2RQL(object):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   382
    """translate a local rql query to be executed on a distant repository"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   383
    def __init__(self, source):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   384
        self.source = source
390
739d12586b9d avoid potentialy mis-interpreted attribute error
sylvain.thenault@logilab.fr
parents: 257
diff changeset
   385
        self.current_operator = None
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   386
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   387
    def _accept_children(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   388
        res = []
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   389
        for child in node.children:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   390
            rql = child.accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   391
            if rql is not None:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   392
                res.append(rql)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   393
        return res
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   394
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   395
    def generate(self, session, rqlst, args):
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   396
        self._session = session
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   397
        self.kwargs = args
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   398
        self.need_translation = False
5174
78438ad513ca #759035: Automate addition of eid cachekey in RQL analysis
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 4913
diff changeset
   399
        return self.visit_union(rqlst)
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   400
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   401
    def visit_union(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   402
        s = self._accept_children(node)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   403
        if len(s) > 1:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   404
            return ' UNION '.join('(%s)' % q for q in s)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   405
        return s[0]
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   406
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   407
    def visit_select(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   408
        """return the tree as an encoded rql string"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   409
        self._varmaker = rqlvar_maker(defined=node.defined_vars.copy())
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   410
        self._const_var = {}
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   411
        if node.distinct:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   412
            base = 'DISTINCT Any'
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   413
        else:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   414
            base = 'Any'
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   415
        s = ['%s %s' % (base, ','.join(v.accept(self) for v in node.selection))]
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   416
        if node.groupby:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   417
            s.append('GROUPBY %s' % ', '.join(group.accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   418
                                              for group in node.groupby))
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   419
        if node.orderby:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   420
            s.append('ORDERBY %s' % ', '.join(self.visit_sortterm(term)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   421
                                              for term in node.orderby))
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   422
        if node.limit is not None:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   423
            s.append('LIMIT %s' % node.limit)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   424
        if node.offset:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   425
            s.append('OFFSET %s' % node.offset)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   426
        restrictions = []
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   427
        if node.where is not None:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   428
            nr = node.where.accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   429
            if nr is not None:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   430
                restrictions.append(nr)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   431
        if restrictions:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   432
            s.append('WHERE %s' % ','.join(restrictions))
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   433
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   434
        if node.having:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   435
            s.append('HAVING %s' % ', '.join(term.accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   436
                                             for term in node.having))
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   437
        subqueries = []
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   438
        for subquery in node.with_:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   439
            subqueries.append('%s BEING (%s)' % (','.join(ca.name for ca in subquery.aliases),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   440
                                                 self.visit_union(subquery.query)))
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   441
        if subqueries:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   442
            s.append('WITH %s' % (','.join(subqueries)))
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   443
        return ' '.join(s)
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   444
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   445
    def visit_and(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   446
        res = self._accept_children(node)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   447
        if res:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   448
            return ', '.join(res)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   449
        return
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   450
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   451
    def visit_or(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   452
        res = self._accept_children(node)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   453
        if len(res) > 1:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   454
            return ' OR '.join('(%s)' % rql for rql in res)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   455
        elif res:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   456
            return res[0]
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   457
        return
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   458
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   459
    def visit_not(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   460
        rql = node.children[0].accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   461
        if rql:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   462
            return 'NOT (%s)' % rql
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   463
        return
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   464
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   465
    def visit_exists(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   466
        return 'EXISTS(%s)' % node.children[0].accept(self)
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   467
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   468
    def visit_relation(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   469
        try:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   470
            if isinstance(node.children[0], Constant):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   471
                # simplified rqlst, reintroduce eid relation
391
42693fe3ef6f skip neged relation here as well
sylvain.thenault@logilab.fr
parents: 390
diff changeset
   472
                try:
42693fe3ef6f skip neged relation here as well
sylvain.thenault@logilab.fr
parents: 390
diff changeset
   473
                    restr, lhs = self.process_eid_const(node.children[0])
42693fe3ef6f skip neged relation here as well
sylvain.thenault@logilab.fr
parents: 390
diff changeset
   474
                except UnknownEid:
42693fe3ef6f skip neged relation here as well
sylvain.thenault@logilab.fr
parents: 390
diff changeset
   475
                    # can safely skip not relation with an unsupported eid
42693fe3ef6f skip neged relation here as well
sylvain.thenault@logilab.fr
parents: 390
diff changeset
   476
                    if node.neged(strict=True):
42693fe3ef6f skip neged relation here as well
sylvain.thenault@logilab.fr
parents: 390
diff changeset
   477
                        return
42693fe3ef6f skip neged relation here as well
sylvain.thenault@logilab.fr
parents: 390
diff changeset
   478
                    raise
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   479
            else:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   480
                lhs = node.children[0].accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   481
                restr = None
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   482
        except UnknownEid:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   483
            # can safely skip not relation with an unsupported eid
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   484
            if node.neged(strict=True):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   485
                return
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   486
            # XXX what about optional relation or outer NOT EXISTS()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   487
            raise
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   488
        if node.optional in ('left', 'both'):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   489
            lhs += '?'
3689
deb13e88e037 follow yams 0.25 api changes to improve performance
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 3605
diff changeset
   490
        if node.r_type == 'eid' or not self.source.schema.rschema(node.r_type).final:
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   491
            self.need_translation = True
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   492
            self.current_operator = node.operator()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   493
            if isinstance(node.children[0], Constant):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   494
                self.current_etypes = (node.children[0].uidtype,)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   495
            else:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   496
                self.current_etypes = node.children[0].variable.stinfo['possibletypes']
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   497
        try:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   498
            rhs = node.children[1].accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   499
        except UnknownEid:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   500
            # can safely skip not relation with an unsupported eid
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   501
            if node.neged(strict=True):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   502
                return
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   503
            # XXX what about optional relation or outer NOT EXISTS()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   504
            raise
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   505
        except ReplaceByInOperator, ex:
1952
8e19c813750d fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1805
diff changeset
   506
            rhs = 'IN (%s)' % ','.join(eid for eid in ex.eids)
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   507
        self.need_translation = False
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   508
        self.current_operator = None
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   509
        if node.optional in ('right', 'both'):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   510
            rhs += '?'
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   511
        if restr is not None:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   512
            return '%s %s %s, %s' % (lhs, node.r_type, rhs, restr)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   513
        return '%s %s %s' % (lhs, node.r_type, rhs)
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   514
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   515
    def visit_comparison(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   516
        if node.operator in ('=', 'IS'):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   517
            return node.children[0].accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   518
        return '%s %s' % (node.operator.encode(),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   519
                          node.children[0].accept(self))
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   520
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   521
    def visit_mathexpression(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   522
        return '(%s %s %s)' % (node.children[0].accept(self),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   523
                               node.operator.encode(),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   524
                               node.children[1].accept(self))
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   525
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   526
    def visit_function(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   527
        #if node.name == 'IN':
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   528
        res = []
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   529
        for child in node.children:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   530
            try:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   531
                rql = child.accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   532
            except UnknownEid, ex:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   533
                continue
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   534
            res.append(rql)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   535
        if not res:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   536
            raise ex
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   537
        return '%s(%s)' % (node.name, ', '.join(res))
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   538
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   539
    def visit_constant(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   540
        if self.need_translation or node.uidtype:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   541
            if node.type == 'Int':
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   542
                return str(self.eid2extid(node.value))
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   543
            if node.type == 'Substitute':
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   544
                key = node.value
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   545
                # ensure we have not yet translated the value...
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   546
                if not key in self._const_var:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   547
                    self.kwargs[key] = self.eid2extid(self.kwargs[key])
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   548
                    self._const_var[key] = None
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   549
        return node.as_string()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   550
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   551
    def visit_variableref(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   552
        """get the sql name for a variable reference"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   553
        return node.name
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   554
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   555
    def visit_sortterm(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   556
        if node.asc:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   557
            return node.term.accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   558
        return '%s DESC' % node.term.accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   559
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   560
    def process_eid_const(self, const):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   561
        value = const.eval(self.kwargs)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   562
        try:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   563
            return None, self._const_var[value]
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   564
        except:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   565
            var = self._varmaker.next()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   566
            self.need_translation = True
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   567
            restr = '%s eid %s' % (var, self.visit_constant(const))
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   568
            self.need_translation = False
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   569
            self._const_var[value] = var
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   570
            return restr, var
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   571
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   572
    def eid2extid(self, eid):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   573
        try:
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   574
            return self.source.eid2extid(eid, self._session)
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   575
        except UnknownEid:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   576
            operator = self.current_operator
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   577
            if operator is not None and operator != '=':
1954
9b20f3504af8 cleanup
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1952
diff changeset
   578
                # deal with query like "X eid > 12"
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   579
                #
1954
9b20f3504af8 cleanup
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1952
diff changeset
   580
                # The problem is that eid order in the external source may
9b20f3504af8 cleanup
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1952
diff changeset
   581
                # differ from the local source
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   582
                #
1954
9b20f3504af8 cleanup
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1952
diff changeset
   583
                # So search for all eids from this source matching the condition
9b20f3504af8 cleanup
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1952
diff changeset
   584
                # locally and then to replace the "> 12" branch by "IN (eids)"
9b20f3504af8 cleanup
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1952
diff changeset
   585
                #
9b20f3504af8 cleanup
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1952
diff changeset
   586
                # XXX we may have to insert a huge number of eids...)
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   587
                sql = "SELECT extid FROM entities WHERE source='%s' AND type IN (%s) AND eid%s%s"
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   588
                etypes = ','.join("'%s'" % etype for etype in self.current_etypes)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   589
                cu = self._session.system_sql(sql % (self.source.uri, etypes,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   590
                                                      operator, eid))
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   591
                # XXX buggy cu.rowcount which may be zero while there are some
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   592
                # results
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   593
                rows = cu.fetchall()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   594
                if rows:
1952
8e19c813750d fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 1805
diff changeset
   595
                    raise ReplaceByInOperator((b64decode(r[0]) for r in rows))
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   596
            raise
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   597