server/sources/pyrorql.py
author sylvain.thenault@logilab.fr
Wed, 13 May 2009 17:01:16 +0200
branchtls-sprint
changeset 1791 c77629112437
parent 1398 5fe84a5f7035
child 1805 7bad2785d167
permissions -rw-r--r--
clear cache before synchronization
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     1
"""Source to query another RQL repository using pyro
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     2
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     3
:organization: Logilab
1238
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 520
diff changeset
     4
:copyright: 2007-2009 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     5
:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     6
"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     7
__docformat__ = "restructuredtext en"
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     8
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     9
import threading
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    10
from os.path import join
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    11
1016
26387b836099 use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents: 520
diff changeset
    12
from time import mktime
26387b836099 use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents: 520
diff changeset
    13
from datetime import datetime
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    14
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    15
from Pyro.errors import PyroError, ConnectionClosedError
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    16
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    17
from logilab.common.configuration import REQUIRED
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    18
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    19
from rql.nodes import Constant
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    20
from rql.utils import rqlvar_maker
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    21
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    22
from cubicweb import dbapi, server
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    23
from cubicweb import BadConnectionId, UnknownEid, ConnectionError
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    24
from cubicweb.cwconfig import register_persistent_options
1238
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 520
diff changeset
    25
from cubicweb.server.sources import AbstractSource, ConnectionWrapper, TimedCache
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    26
1138
22f634977c95 make pylint happy, fix some bugs on the way
sylvain.thenault@logilab.fr
parents: 1016
diff changeset
    27
class ReplaceByInOperator(Exception):
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    28
    def __init__(self, eids):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    29
        self.eids = eids
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
    30
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    31
class PyroRQLSource(AbstractSource):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    32
    """External repository source, using Pyro connection"""
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
    33
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    34
    # boolean telling if modification hooks should be called when something is
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    35
    # modified in this source
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    36
    should_call_hooks = False
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    37
    # boolean telling if the repository should connect to this source during
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    38
    # migration
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    39
    connect_for_migration = False
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
    40
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    41
    support_entities = None
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
    42
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    43
    options = (
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    44
        # XXX pyro-ns host/port
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    45
        ('pyro-ns-id',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    46
         {'type' : 'string',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    47
          'default': REQUIRED,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    48
          'help': 'identifier of the repository in the pyro name server',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    49
          'group': 'pyro-source', 'inputlevel': 0,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    50
          }),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    51
        ('mapping-file',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    52
         {'type' : 'string',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    53
          'default': REQUIRED,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    54
          'help': 'path to a python file with the schema mapping definition',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    55
          'group': 'pyro-source', 'inputlevel': 1,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    56
          }),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    57
        ('cubicweb-user',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    58
         {'type' : 'string',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    59
          'default': REQUIRED,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    60
          'help': 'user to use for connection on the distant repository',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    61
          'group': 'pyro-source', 'inputlevel': 0,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    62
          }),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    63
        ('cubicweb-password',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    64
         {'type' : 'password',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    65
          'default': '',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    66
          'help': 'user to use for connection on the distant repository',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    67
          'group': 'pyro-source', 'inputlevel': 0,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    68
          }),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    69
        ('base-url',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    70
         {'type' : 'string',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    71
          'default': '',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    72
          'help': 'url of the web site for the distant repository, if you want '
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    73
          'to generate external link to entities from this repository',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    74
          'group': 'pyro-source', 'inputlevel': 1,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    75
          }),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    76
        ('pyro-ns-host',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    77
         {'type' : 'string',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    78
          'default': None,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    79
          'help': 'Pyro name server\'s host. If not set, default to the value \
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    80
from all_in_one.conf.',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    81
          'group': 'pyro-source', 'inputlevel': 1,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    82
          }),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    83
        ('pyro-ns-port',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    84
         {'type' : 'int',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    85
          'default': None,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    86
          'help': 'Pyro name server\'s listening port. If not set, default to \
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    87
the value from all_in_one.conf.',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    88
          'group': 'pyro-source', 'inputlevel': 1,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    89
          }),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    90
        ('pyro-ns-group',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    91
         {'type' : 'string',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    92
          'default': None,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    93
          'help': 'Pyro name server\'s group where the repository will be \
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    94
registered. If not set, default to the value from all_in_one.conf.',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    95
          'group': 'pyro-source', 'inputlevel': 1,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    96
          }),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    97
        ('synchronization-interval',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    98
         {'type' : 'int',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    99
          'default': 5*60,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   100
          'help': 'interval between synchronization with the external \
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   101
repository (default to 5 minutes).',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   102
          'group': 'pyro-source', 'inputlevel': 2,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   103
          }),
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   104
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   105
    )
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   106
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   107
    PUBLIC_KEYS = AbstractSource.PUBLIC_KEYS + ('base-url',)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   108
    _conn = None
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   109
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   110
    def __init__(self, repo, appschema, source_config, *args, **kwargs):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   111
        AbstractSource.__init__(self, repo, appschema, source_config,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   112
                                *args, **kwargs)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   113
        mappingfile = source_config['mapping-file']
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   114
        if not mappingfile[0] == '/':
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   115
            mappingfile = join(repo.config.apphome, mappingfile)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   116
        mapping = {}
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   117
        execfile(mappingfile, mapping)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   118
        self.support_entities = mapping['support_entities']
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   119
        self.support_relations = mapping.get('support_relations', {})
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   120
        self.dont_cross_relations = mapping.get('dont_cross_relations', ())
417
828424d2eb3c oops, get back cross_relation from mapping to rql source
sylvain.thenault@logilab.fr
parents: 391
diff changeset
   121
        self.cross_relations = mapping.get('cross_relations', ())
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   122
        baseurl = source_config.get('base-url')
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   123
        if baseurl and not baseurl.endswith('/'):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   124
            source_config['base-url'] += '/'
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   125
        self.config = source_config
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   126
        myoptions = (('%s.latest-update-time' % self.uri,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   127
                      {'type' : 'int', 'sitewide': True,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   128
                       'default': 0,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   129
                       'help': _('timestamp of the latest source synchronization.'),
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   130
                       'group': 'sources',
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   131
                       }),)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   132
        register_persistent_options(myoptions)
1238
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   133
        self._query_cache = TimedCache(30)
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   134
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   135
    def last_update_time(self):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   136
        pkey = u'sources.%s.latest-update-time' % self.uri
1398
5fe84a5f7035 rename internal entity types to have CW prefix instead of E
sylvain.thenault@logilab.fr
parents: 1263
diff changeset
   137
        rql = 'Any V WHERE X is CWProperty, X value V, X pkey %(k)s'
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   138
        session = self.repo.internal_session()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   139
        try:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   140
            rset = session.execute(rql, {'k': pkey})
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   141
            if not rset:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   142
                # insert it
1398
5fe84a5f7035 rename internal entity types to have CW prefix instead of E
sylvain.thenault@logilab.fr
parents: 1263
diff changeset
   143
                session.execute('INSERT CWProperty X: X pkey %(k)s, X value %(v)s',
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   144
                                {'k': pkey, 'v': u'0'})
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   145
                session.commit()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   146
                timestamp = 0
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   147
            else:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   148
                assert len(rset) == 1
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   149
                timestamp = int(rset[0][0])
1016
26387b836099 use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   150
            return datetime.fromtimestamp(timestamp)
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   151
        finally:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   152
            session.close()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   153
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   154
    def init(self):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   155
        """method called by the repository once ready to handle request"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   156
        interval = int(self.config.get('synchronization-interval', 5*60))
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   157
        self.repo.looping_task(interval, self.synchronize)
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   158
        self.repo.looping_task(self._query_cache.ttl.seconds/10, self._query_cache.clear_expired)
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   159
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   160
    def synchronize(self, mtime=None):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   161
        """synchronize content known by this repository with content in the
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   162
        external repository
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   163
        """
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   164
        self.info('synchronizing pyro source %s', self.uri)
519
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   165
        cnx = self.get_connection()
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   166
        extrepo = cnx._repo
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   167
        etypes = self.support_entities.keys()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   168
        if mtime is None:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   169
            mtime = self.last_update_time()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   170
        updatetime, modified, deleted = extrepo.entities_modified_since(etypes,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   171
                                                                        mtime)
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   172
        self._query_cache.clear()
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   173
        repo = self.repo
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   174
        session = repo.internal_session()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   175
        try:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   176
            for etype, extid in modified:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   177
                try:
519
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   178
                    exturi = cnx.describe(extid)[1]
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   179
                    if exturi == 'system' or not exturi in repo.sources_by_uri:
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   180
                        eid = self.extid2eid(extid, etype, session)
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   181
                        rset = session.eid_rset(eid, etype)
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   182
                        entity = rset.get_entity(0, 0)
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   183
                        entity.complete(entity.e_schema.indexable_attributes())
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   184
                        repo.index_entity(session, entity)
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   185
                except:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   186
                    self.exception('while updating %s with external id %s of source %s',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   187
                                   etype, extid, self.uri)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   188
                    continue
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   189
            for etype, extid in deleted:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   190
                try:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   191
                    eid = self.extid2eid(extid, etype, session, insert=False)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   192
                    # entity has been deleted from external repository but is not known here
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   193
                    if eid is not None:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   194
                        repo.delete_info(session, eid)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   195
                except:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   196
                    self.exception('while updating %s with external id %s of source %s',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   197
                                   etype, extid, self.uri)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   198
                    continue
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   199
            session.execute('SET X value %(v)s WHERE X pkey %(k)s',
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   200
                            {'k': u'sources.%s.latest-update-time' % self.uri,
1016
26387b836099 use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   201
                             'v': unicode(int(mktime(updatetime.timetuple())))})
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   202
            session.commit()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   203
        finally:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   204
            session.close()
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   205
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   206
    def _get_connection(self):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   207
        """open and return a connection to the source"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   208
        nshost = self.config.get('pyro-ns-host') or self.repo.config['pyro-ns-host']
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   209
        nsport = self.config.get('pyro-ns-port') or self.repo.config['pyro-ns-port']
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   210
        nsgroup = self.config.get('pyro-ns-group') or self.repo.config['pyro-ns-group']
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   211
        #cnxprops = ConnectionProperties(cnxtype=self.config['cnx-type'])
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   212
        return dbapi.connect(database=self.config['pyro-ns-id'],
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   213
                             user=self.config['cubicweb-user'],
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   214
                             password=self.config['cubicweb-password'],
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   215
                             host=nshost, port=nsport, group=nsgroup,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   216
                             setvreg=False) #cnxprops=cnxprops)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   217
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   218
    def get_connection(self):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   219
        try:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   220
            return self._get_connection()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   221
        except (ConnectionError, PyroError):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   222
            self.critical("can't get connection to source %s", self.uri,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   223
                          exc_info=1)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   224
            return ConnectionWrapper()
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   225
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   226
    def check_connection(self, cnx):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   227
        """check connection validity, return None if the connection is still valid
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   228
        else a new connection
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   229
        """
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   230
        # we have to transfer manually thread ownership. This can be done safely
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   231
        # since the pool to which belong the connection is affected to one
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   232
        # session/thread and can't be called simultaneously
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   233
        try:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   234
            cnx._repo._transferThread(threading.currentThread())
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   235
        except AttributeError:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   236
            # inmemory connection
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   237
            pass
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   238
        if not isinstance(cnx, ConnectionWrapper):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   239
            try:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   240
                cnx.check()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   241
                return # ok
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   242
            except (BadConnectionId, ConnectionClosedError):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   243
                pass
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   244
        # try to reconnect
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   245
        return self.get_connection()
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   246
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   247
    def syntax_tree_search(self, session, union, args=None, cachekey=None,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   248
                           varmap=None):
1239
7eda09aa7553 remove debug stuff
sylvain.thenault@logilab.fr
parents: 1238
diff changeset
   249
        #assert not varmap, (varmap, union)
1238
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   250
        rqlkey = union.as_string(kwargs=args)
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   251
        try:
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   252
            results = self._query_cache[rqlkey]
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   253
        except KeyError:
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   254
            results = self._syntax_tree_search(session, union, args)
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   255
            self._query_cache[rqlkey] = results
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   256
        return results
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   257
1238
fa29b5b60107 set 30sec query cache on pyro source, important speedup for pages generating multiple time the same external query
sylvain.thenault@logilab.fr
parents: 520
diff changeset
   258
    def _syntax_tree_search(self, session, union, args):
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   259
        """return result from this source for a rql query (actually from a rql
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   260
        syntax tree and a solution dictionary mapping each used variable to a
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   261
        possible type). If cachekey is given, the query necessary to fetch the
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   262
        results (but not the results themselves) may be cached using this key.
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   263
        """
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   264
        if not args is None:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   265
            args = args.copy()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   266
        if server.DEBUG:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   267
            print 'RQL FOR PYRO SOURCE', self.uri
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   268
            print union.as_string()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   269
            if args: print 'ARGS', args
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   270
            print 'SOLUTIONS', ','.join(str(s.solutions) for s in union.children)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   271
        # get cached cursor anyway
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   272
        cu = session.pool[self.uri]
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   273
        if cu is None:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   274
            # this is a ConnectionWrapper instance
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   275
            msg = session._("can't connect to source %s, some data may be missing")
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   276
            session.set_shared_data('sources_error', msg % self.uri)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   277
            return []
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   278
        try:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   279
            rql, cachekey = RQL2RQL(self).generate(session, union, args)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   280
        except UnknownEid, ex:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   281
            if server.DEBUG:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   282
                print 'unknown eid', ex, 'no results'
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   283
            return []
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   284
        if server.DEBUG:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   285
            print 'TRANSLATED RQL', rql
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   286
        try:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   287
            rset = cu.execute(rql, args, cachekey)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   288
        except Exception, ex:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   289
            self.exception(str(ex))
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   290
            msg = session._("error while querying source %s, some data may be missing")
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   291
            session.set_shared_data('sources_error', msg % self.uri)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   292
            return []
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   293
        descr = rset.description
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   294
        if rset:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   295
            needtranslation = []
519
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   296
            rows = rset.rows
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   297
            for i, etype in enumerate(descr[0]):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   298
                if (etype is None or not self.schema.eschema(etype).is_final() or
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   299
                    getattr(union.locate_subquery(i, etype, args).selection[i], 'uidtype', None)):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   300
                    needtranslation.append(i)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   301
            if needtranslation:
519
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   302
                cnx = session.pool.connection(self.uri)
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   303
                for rowindex in xrange(rset.rowcount - 1, -1, -1):
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   304
                    row = rows[rowindex]
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   305
                    for colindex in needtranslation:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   306
                        if row[colindex] is not None: # optional variable
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   307
                            etype = descr[rowindex][colindex]
519
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   308
                            exttype, exturi, extid = cnx.describe(row[colindex])
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   309
                            if exturi == 'system' or not exturi in self.repo.sources_by_uri:
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   310
                                eid = self.extid2eid(row[colindex], etype, session)
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   311
                                row[colindex] = eid
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   312
                            else:
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   313
                                # skip this row
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   314
                                del rows[rowindex]
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   315
                                del descr[rowindex]
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   316
                                break
06390418cd9a pyrorql source now ignore external eids which are themselves coming from another external source already in use by the repository (should have the same uri)
sylvain.thenault@logilab.fr
parents: 417
diff changeset
   317
            results = rows
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   318
        else:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   319
            results = []
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   320
        if server.DEBUG:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   321
            if len(results)>10:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   322
                print '--------------->', results[:10], '...', len(results)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   323
            else:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   324
                print '--------------->', results
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   325
        return results
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   326
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   327
    def _entity_relations_and_kwargs(self, session, entity):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   328
        relations = []
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   329
        kwargs = {'x': self.eid2extid(entity.eid, session)}
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   330
        for key, val in entity.iteritems():
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   331
            relations.append('X %s %%(%s)s' % (key, key))
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   332
            kwargs[key] = val
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   333
        return relations, kwargs
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   334
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   335
    def add_entity(self, session, entity):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   336
        """add a new entity to the source"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   337
        raise NotImplementedError()
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   338
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   339
    def update_entity(self, session, entity):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   340
        """update an entity in the source"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   341
        relations, kwargs = self._entity_relations_and_kwargs(session, entity)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   342
        cu = session.pool[self.uri]
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   343
        cu.execute('SET %s WHERE X eid %%(x)s' % ','.join(relations),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   344
                   kwargs, 'x')
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   345
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   346
    def delete_entity(self, session, etype, eid):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   347
        """delete an entity from the source"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   348
        cu = session.pool[self.uri]
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   349
        cu.execute('DELETE %s X WHERE X eid %%(x)s' % etype,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   350
                   {'x': self.eid2extid(eid, session)}, 'x')
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   351
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   352
    def add_relation(self, session, subject, rtype, object):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   353
        """add a relation to the source"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   354
        cu = session.pool[self.uri]
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   355
        cu.execute('SET X %s Y WHERE X eid %%(x)s, Y eid %%(y)s' % rtype,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   356
                   {'x': self.eid2extid(subject, session),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   357
                    'y': self.eid2extid(object, session)}, ('x', 'y'))
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   358
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   359
    def delete_relation(self, session, subject, rtype, object):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   360
        """delete a relation from the source"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   361
        cu = session.pool[self.uri]
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   362
        cu.execute('DELETE X %s Y WHERE X eid %%(x)s, Y eid %%(y)s' % rtype,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   363
                   {'x': self.eid2extid(subject, session),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   364
                    'y': self.eid2extid(object, session)}, ('x', 'y'))
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   365
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   366
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   367
class RQL2RQL(object):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   368
    """translate a local rql query to be executed on a distant repository"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   369
    def __init__(self, source):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   370
        self.source = source
390
739d12586b9d avoid potentialy mis-interpreted attribute error
sylvain.thenault@logilab.fr
parents: 257
diff changeset
   371
        self.current_operator = None
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   372
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   373
    def _accept_children(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   374
        res = []
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   375
        for child in node.children:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   376
            rql = child.accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   377
            if rql is not None:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   378
                res.append(rql)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   379
        return res
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   380
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   381
    def generate(self, session, rqlst, args):
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   382
        self._session = session
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   383
        self.kwargs = args
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   384
        self.cachekey = []
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   385
        self.need_translation = False
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   386
        return self.visit_union(rqlst), self.cachekey
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   387
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   388
    def visit_union(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   389
        s = self._accept_children(node)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   390
        if len(s) > 1:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   391
            return ' UNION '.join('(%s)' % q for q in s)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   392
        return s[0]
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   393
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   394
    def visit_select(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   395
        """return the tree as an encoded rql string"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   396
        self._varmaker = rqlvar_maker(defined=node.defined_vars.copy())
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   397
        self._const_var = {}
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   398
        if node.distinct:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   399
            base = 'DISTINCT Any'
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   400
        else:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   401
            base = 'Any'
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   402
        s = ['%s %s' % (base, ','.join(v.accept(self) for v in node.selection))]
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   403
        if node.groupby:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   404
            s.append('GROUPBY %s' % ', '.join(group.accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   405
                                              for group in node.groupby))
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   406
        if node.orderby:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   407
            s.append('ORDERBY %s' % ', '.join(self.visit_sortterm(term)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   408
                                              for term in node.orderby))
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   409
        if node.limit is not None:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   410
            s.append('LIMIT %s' % node.limit)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   411
        if node.offset:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   412
            s.append('OFFSET %s' % node.offset)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   413
        restrictions = []
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   414
        if node.where is not None:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   415
            nr = node.where.accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   416
            if nr is not None:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   417
                restrictions.append(nr)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   418
        if restrictions:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   419
            s.append('WHERE %s' % ','.join(restrictions))
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   420
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   421
        if node.having:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   422
            s.append('HAVING %s' % ', '.join(term.accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   423
                                             for term in node.having))
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   424
        subqueries = []
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   425
        for subquery in node.with_:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   426
            subqueries.append('%s BEING (%s)' % (','.join(ca.name for ca in subquery.aliases),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   427
                                                 self.visit_union(subquery.query)))
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   428
        if subqueries:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   429
            s.append('WITH %s' % (','.join(subqueries)))
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   430
        return ' '.join(s)
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   431
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   432
    def visit_and(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   433
        res = self._accept_children(node)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   434
        if res:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   435
            return ', '.join(res)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   436
        return
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   437
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   438
    def visit_or(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   439
        res = self._accept_children(node)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   440
        if len(res) > 1:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   441
            return ' OR '.join('(%s)' % rql for rql in res)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   442
        elif res:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   443
            return res[0]
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   444
        return
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   445
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   446
    def visit_not(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   447
        rql = node.children[0].accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   448
        if rql:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   449
            return 'NOT (%s)' % rql
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   450
        return
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   451
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   452
    def visit_exists(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   453
        return 'EXISTS(%s)' % node.children[0].accept(self)
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   454
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   455
    def visit_relation(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   456
        try:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   457
            if isinstance(node.children[0], Constant):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   458
                # simplified rqlst, reintroduce eid relation
391
42693fe3ef6f skip neged relation here as well
sylvain.thenault@logilab.fr
parents: 390
diff changeset
   459
                try:
42693fe3ef6f skip neged relation here as well
sylvain.thenault@logilab.fr
parents: 390
diff changeset
   460
                    restr, lhs = self.process_eid_const(node.children[0])
42693fe3ef6f skip neged relation here as well
sylvain.thenault@logilab.fr
parents: 390
diff changeset
   461
                except UnknownEid:
42693fe3ef6f skip neged relation here as well
sylvain.thenault@logilab.fr
parents: 390
diff changeset
   462
                    # can safely skip not relation with an unsupported eid
42693fe3ef6f skip neged relation here as well
sylvain.thenault@logilab.fr
parents: 390
diff changeset
   463
                    if node.neged(strict=True):
42693fe3ef6f skip neged relation here as well
sylvain.thenault@logilab.fr
parents: 390
diff changeset
   464
                        return
42693fe3ef6f skip neged relation here as well
sylvain.thenault@logilab.fr
parents: 390
diff changeset
   465
                    raise
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   466
            else:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   467
                lhs = node.children[0].accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   468
                restr = None
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   469
        except UnknownEid:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   470
            # can safely skip not relation with an unsupported eid
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   471
            if node.neged(strict=True):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   472
                return
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   473
            # XXX what about optional relation or outer NOT EXISTS()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   474
            raise
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   475
        if node.optional in ('left', 'both'):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   476
            lhs += '?'
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   477
        if node.r_type == 'eid' or not self.source.schema.rschema(node.r_type).is_final():
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   478
            self.need_translation = True
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   479
            self.current_operator = node.operator()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   480
            if isinstance(node.children[0], Constant):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   481
                self.current_etypes = (node.children[0].uidtype,)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   482
            else:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   483
                self.current_etypes = node.children[0].variable.stinfo['possibletypes']
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   484
        try:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   485
            rhs = node.children[1].accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   486
        except UnknownEid:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   487
            # can safely skip not relation with an unsupported eid
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   488
            if node.neged(strict=True):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   489
                return
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   490
            # XXX what about optional relation or outer NOT EXISTS()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   491
            raise
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   492
        except ReplaceByInOperator, ex:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   493
            rhs = 'IN (%s)' % ','.join(str(eid) for eid in ex.eids)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   494
        self.need_translation = False
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   495
        self.current_operator = None
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   496
        if node.optional in ('right', 'both'):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   497
            rhs += '?'
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   498
        if restr is not None:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   499
            return '%s %s %s, %s' % (lhs, node.r_type, rhs, restr)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   500
        return '%s %s %s' % (lhs, node.r_type, rhs)
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   501
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   502
    def visit_comparison(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   503
        if node.operator in ('=', 'IS'):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   504
            return node.children[0].accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   505
        return '%s %s' % (node.operator.encode(),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   506
                          node.children[0].accept(self))
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   507
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   508
    def visit_mathexpression(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   509
        return '(%s %s %s)' % (node.children[0].accept(self),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   510
                               node.operator.encode(),
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   511
                               node.children[1].accept(self))
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   512
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   513
    def visit_function(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   514
        #if node.name == 'IN':
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   515
        res = []
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   516
        for child in node.children:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   517
            try:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   518
                rql = child.accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   519
            except UnknownEid, ex:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   520
                continue
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   521
            res.append(rql)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   522
        if not res:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   523
            raise ex
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   524
        return '%s(%s)' % (node.name, ', '.join(res))
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   525
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   526
    def visit_constant(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   527
        if self.need_translation or node.uidtype:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   528
            if node.type == 'Int':
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   529
                return str(self.eid2extid(node.value))
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   530
            if node.type == 'Substitute':
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   531
                key = node.value
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   532
                # ensure we have not yet translated the value...
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   533
                if not key in self._const_var:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   534
                    self.kwargs[key] = self.eid2extid(self.kwargs[key])
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   535
                    self.cachekey.append(key)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   536
                    self._const_var[key] = None
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   537
        return node.as_string()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   538
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   539
    def visit_variableref(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   540
        """get the sql name for a variable reference"""
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   541
        return node.name
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   542
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   543
    def visit_sortterm(self, node):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   544
        if node.asc:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   545
            return node.term.accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   546
        return '%s DESC' % node.term.accept(self)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   547
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   548
    def process_eid_const(self, const):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   549
        value = const.eval(self.kwargs)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   550
        try:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   551
            return None, self._const_var[value]
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   552
        except:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   553
            var = self._varmaker.next()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   554
            self.need_translation = True
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   555
            restr = '%s eid %s' % (var, self.visit_constant(const))
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   556
            self.need_translation = False
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   557
            self._const_var[value] = var
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   558
            return restr, var
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   559
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   560
    def eid2extid(self, eid):
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   561
        try:
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   562
            return self.source.eid2extid(eid, self._session)
257
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   563
        except UnknownEid:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   564
            operator = self.current_operator
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   565
            if operator is not None and operator != '=':
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   566
                # deal with query like X eid > 12
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   567
                #
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   568
                # The problem is
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   569
                # that eid order in the external source may differ from the
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   570
                # local source
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   571
                #
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   572
                # So search for all eids from this
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   573
                # source matching the condition locally and then to replace the
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   574
                # > 12 branch by IN (eids) (XXX we may have to insert a huge
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   575
                # number of eids...)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   576
                # planner so that
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   577
                sql = "SELECT extid FROM entities WHERE source='%s' AND type IN (%s) AND eid%s%s"
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   578
                etypes = ','.join("'%s'" % etype for etype in self.current_etypes)
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   579
                cu = self._session.system_sql(sql % (self.source.uri, etypes,
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   580
                                                      operator, eid))
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   581
                # XXX buggy cu.rowcount which may be zero while there are some
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   582
                # results
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   583
                rows = cu.fetchall()
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   584
                if rows:
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   585
                    raise ReplaceByInOperator((r[0] for r in rows))
4c7d3af7e94d restore multi-sources capabilities
Sylvain Thenault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   586
            raise
1791
c77629112437 clear cache before synchronization
sylvain.thenault@logilab.fr
parents: 1398
diff changeset
   587