"""Defines the central class for the CubicWeb RQL server: the repository.

The repository is an abstraction allowing execution of rql queries against

data sources. Most of the work is actually done in helper classes. The

repository mainly:

* brings these classes all together to provide a single access

  point to a cubicweb instance.

* handles session management

* provides method for pyro registration, to call if pyro is enabled

:organization: Logilab

:copyright: 2001-2009 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2.

:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr

:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses

"""

__docformat__ = "restructuredtext en"

import sys

import Queue

from os.path import join, exists

from datetime import datetime

from time import time, localtime, strftime

from logilab.common.decorators import cached

from yams import BadSchemaDefinition

from rql import RQLSyntaxError

from cubicweb import (CW_SOFTWARE_ROOT, UnknownEid, AuthenticationError,

                      ETypeNotSupportedBySources, RTypeNotSupportedBySources,

                      BadConnectionId, Unauthorized, ValidationError,

                      ExecutionError, typed_eid,

                      CW_MIGRATION_MAP)

from cubicweb.cwvreg import CubicWebVRegistry

from cubicweb.schema import VIRTUAL_RTYPES, CubicWebSchema

from cubicweb import server

from cubicweb.server.utils import RepoThread, LoopTask

from cubicweb.server.pool import ConnectionsPool, LateOperation, SingleLastOperation

from cubicweb.server.session import Session, InternalSession

from cubicweb.server.querier import QuerierHelper

from cubicweb.server.sources import get_source

from cubicweb.server.hooksmanager import HooksManager

from cubicweb.server.hookhelper import rproperty

class CleanupEidTypeCacheOp(SingleLastOperation):

    """on rollback of a insert query or commit of delete query, we have to

    clear repository's cache from no more valid entries

    NOTE: querier's rqlst/solutions cache may have been polluted too with

    queries such as Any X WHERE X eid 32 if 32 has been rollbacked however

    generated queries are unpredictable and analysing all the cache probably

    too expensive. Notice that there is no pb when using args to specify eids

    instead of giving them into the rql string.

    """

    def commit_event(self):

        """the observed connections pool has been rollbacked,

        remove inserted eid from repository type/source cache

        """

        try:

            self.repo.clear_caches(self.session.transaction_data['pendingeids'])

        except KeyError:

            pass

    def rollback_event(self):

        """the observed connections pool has been rollbacked,

        remove inserted eid from repository type/source cache

        """

        try:

            self.repo.clear_caches(self.session.transaction_data['neweids'])

        except KeyError:

            pass

class FTIndexEntityOp(LateOperation):

    """operation to delay entity full text indexation to commit

    since fti indexing may trigger discovery of other entities, it should be

    triggered on precommit, not commit, and this should be done after other

    precommit operation which may add relations to the entity

    """

    def precommit_event(self):

        session = self.session

        entity = self.entity

        if entity.eid in session.transaction_data.get('pendingeids', ()):

            return # entity added and deleted in the same transaction

        session.repo.system_source.fti_unindex_entity(session, entity.eid)

        for container in entity.fti_containers():

            session.repo.index_entity(session, container)

    def commit_event(self):

        pass

def del_existing_rel_if_needed(session, eidfrom, rtype, eidto):

    """delete existing relation when adding a new one if card is 1 or ?

    have to be done once the new relation has been inserted to avoid having

    an entity without a relation for some time

    this kind of behaviour has to be done in the repository so we don't have

    hooks order hazardness

    """

    # skip delete queries (only?) if session is an internal session. This is

    # hooks responsability to ensure they do not violate relation's cardinality

    if session.is_super_session:

        return

    card = rproperty(session, rtype, eidfrom, eidto, 'cardinality')

    # one may be tented to check for neweids but this may cause more than one

    # relation even with '1?'  cardinality if thoses relations are added in the

    # same transaction where the entity is being created. This never occurs from

    # the web interface but may occurs during test or dbapi connection (though

    # not expected for this).  So: don't do it, we pretend to ensure repository

    # consistency.

    if card[0] in '1?':

        rschema = session.repo.schema.rschema(rtype)

        if not rschema.inlined:

            session.unsafe_execute(

                'DELETE X %s Y WHERE X eid %%(x)s, NOT Y eid %%(y)s' % rtype,

                {'x': eidfrom, 'y': eidto}, 'x')

    if card[1] in '1?':

        session.unsafe_execute(

            'DELETE X %s Y WHERE NOT X eid %%(x)s, Y eid %%(y)s' % rtype,

            {'x': eidfrom, 'y': eidto}, 'y')

class Repository(object):

    """a repository provides access to a set of persistent storages for

    entities and relations

    XXX protect pyro access

    """

    def __init__(self, config, vreg=None, debug=False):

        self.config = config

        if vreg is None:

            vreg = CubicWebVRegistry(config, debug)

        self.vreg = vreg

        self.pyro_registered = False

        self.info('starting repository from %s', self.config.apphome)

        # dictionary of opened sessions

        self._sessions = {}

        # list of functions to be called at regular interval

        self._looping_tasks = []

        # list of running threads

        self._running_threads = []

        # initial schema, should be build or replaced latter

        self.schema = CubicWebSchema(config.appid)

        # querier helper, need to be created after sources initialization

        self.querier = QuerierHelper(self, self.schema)

        # should we reindex in changes?

        self.do_fti = not config['delay-full-text-indexation']

        # sources

        self.sources = []

        self.sources_by_uri = {}

        # FIXME: store additional sources info in the system database ?

        # FIXME: sources should be ordered (add_entity priority)

        for uri, source_config in config.sources().items():

            if uri == 'admin':

                # not an actual source

                continue

            source = self.get_source(uri, source_config)

            self.sources_by_uri[uri] = source

            self.sources.append(source)

        self.system_source = self.sources_by_uri['system']

        # ensure system source is the first one

        self.sources.remove(self.system_source)

        self.sources.insert(0, self.system_source)

        # cache eid -> type / source

        self._type_source_cache = {}

        # cache (extid, source uri) -> eid

        self._extid_cache = {}

        # create the hooks manager

        self.hm = HooksManager(self.schema)

        # open some connections pools

        self._available_pools = Queue.Queue()

        self._available_pools.put_nowait(ConnectionsPool(self.sources))

        if config.read_instance_schema:

            # normal start: load the instance schema from the database

            self.fill_schema()

        elif config.bootstrap_schema:

            # usually during repository creation

            self.warning("set fs instance'schema as bootstrap schema")

            config.bootstrap_cubes()

            self.set_bootstrap_schema(self.config.load_schema())

            # need to load the Any and CWUser entity types

            self.vreg.schema = self.schema

            etdirectory = join(CW_SOFTWARE_ROOT, 'entities')

            self.vreg.init_registration([etdirectory])

            self.vreg.load_file(join(etdirectory, '__init__.py'),

                                'cubicweb.entities.__init__')

            self.vreg.load_file(join(etdirectory, 'authobjs.py'),

                                'cubicweb.entities.authobjs')

        else:

            # test start: use the file system schema (quicker)

            self.warning("set fs instance'schema")

            config.bootstrap_cubes()

            self.set_schema(self.config.load_schema())

        if not config.creating:

            if 'CWProperty' in self.schema:

                self.vreg.init_properties(self.properties())

            # call source's init method to complete their initialisation if

            # needed (for instance looking for persistent configuration using an

            # internal session, which is not possible until pools have been

            # initialized)

            for source in self.sources:

                source.init()

        else:

            # call init_creating so for instance native source can configurate

            # tsearch according to postgres version

            for source in self.sources:

                source.init_creating()

        # close initialization pool and reopen fresh ones for proper

        # initialization now that we know cubes

        self._get_pool().close(True)

        # list of available pools (we can't iterated on Queue instance)

        self.pools = []

        for i in xrange(config['connections-pool-size']):

            self.pools.append(ConnectionsPool(self.sources))

            self._available_pools.put_nowait(self.pools[-1])

        self._shutting_down = False

        if not (config.creating or config.repairing):

            # call instance level initialisation hooks

            self.hm.call_hooks('server_startup', repo=self)

            # register a task to cleanup expired session

            self.looping_task(self.config['session-time']/3.,

                              self.clean_sessions)

    # internals ###############################################################

    def get_source(self, uri, source_config):

        source_config['uri'] = uri

        return get_source(source_config, self.schema, self)

    def set_schema(self, schema, resetvreg=True):

        schema.rebuild_infered_relations()

        self.info('set schema %s %#x', schema.name, id(schema))

        self.debug(', '.join(sorted(str(e) for e in schema.entities())))

        self.querier.set_schema(schema)

        for source in self.sources:

            source.set_schema(schema)

        self.schema = schema

        if resetvreg:

            # full reload of all appobjects

            self.vreg.reset()

            self.vreg.set_schema(schema)

        self.hm.set_schema(schema)

        self.hm.register_system_hooks(self.config)

        # instance specific hooks

        if self.config.instance_hooks:

            self.info('loading instance hooks')

            self.hm.register_hooks(self.config.load_hooks(self.vreg))

    def fill_schema(self):

        """lod schema from the repository"""

        from cubicweb.server.schemaserial import deserialize_schema

        self.info('loading schema from the repository')

        appschema = CubicWebSchema(self.config.appid)

        self.set_bootstrap_schema(self.config.load_bootstrap_schema())

        self.debug('deserializing db schema into %s %#x', appschema.name, id(appschema))

        session = self.internal_session()

        try:

            try:

                deserialize_schema(appschema, session)

            except BadSchemaDefinition:

                raise

            except Exception, ex:

                import traceback

                traceback.print_exc()

                raise Exception('Is the database initialised ? (cause: %s)' %

                                (ex.args and ex.args[0].strip() or 'unknown')), \

                                None, sys.exc_info()[-1]

            self.info('set the actual schema')

            # XXX have to do this since CWProperty isn't in the bootstrap schema

            #     it'll be redone in set_schema

            self.set_bootstrap_schema(appschema)

            # 2.49 migration

            if exists(join(self.config.apphome, 'vc.conf')):

                session.set_pool()

                if not 'template' in file(join(self.config.apphome, 'vc.conf')).read():

                    # remaning from cubicweb < 2.38...

                    session.execute('DELETE CWProperty X WHERE X pkey "system.version.template"')

                    session.commit()

        finally:

            session.close()

        self.config.init_cubes(self.get_cubes())

        self.set_schema(appschema)

    def set_bootstrap_schema(self, schema):

        """disable hooks when setting a bootstrap schema, but restore

        the configuration for the next time

        """

        config = self.config

        # XXX refactor

        config.core_hooks = False

        config.usergroup_hooks = False

        config.schema_hooks = False

        config.notification_hooks = False

        config.instance_hooks = False

        self.set_schema(schema, resetvreg=False)

        config.core_hooks = True

        config.usergroup_hooks = True

        config.schema_hooks = True

        config.notification_hooks = True

        config.instance_hooks = True

    def start_looping_tasks(self):

        assert isinstance(self._looping_tasks, list), 'already started'

        for i, (interval, func) in enumerate(self._looping_tasks):

            self._looping_tasks[i] = task = LoopTask(interval, func)

            self.info('starting task %s with interval %.2fs', task.name,

                      interval)

            task.start()

        # ensure no tasks will be further added

        self._looping_tasks = tuple(self._looping_tasks)

    def looping_task(self, interval, func):

        """register a function to be called every `interval` seconds.

        looping tasks can only be registered during repository initialization,

        once done this method will fail.

        """

        try:

            self._looping_tasks.append( (interval, func) )

        except AttributeError:

            raise RuntimeError("can't add looping task once the repository is started")

    def threaded_task(self, func):

        """start function in a separated thread"""

        t = RepoThread(func, self._running_threads)

        t.start()

    #@locked

    def _get_pool(self):

        try:

            return self._available_pools.get(True, timeout=5)

        except Queue.Empty:

            raise Exception('no pool available after 5 secs, probably either a '

                            'bug in code (to many uncommited/rollbacked '

                            'connections) or to much load on the server (in '

                            'which case you can try to set a bigger '

                            'connections pools size)')

    def _free_pool(self, pool):

        self._available_pools.put_nowait(pool)

    def pinfo(self):

        # XXX: session.pool is accessed from a local storage, would be interesting

        #      to see if there is a pool set in any thread specific data)

        import threading

        return '%s: %s (%s)' % (self._available_pools.qsize(),

                                ','.join(session.user.login for session in self._sessions.values()

                                         if session.pool),

                                threading.currentThread())

    def shutdown(self):

        """called on server stop event to properly close opened sessions and

        connections

        """

        self._shutting_down = True

        if isinstance(self._looping_tasks, tuple): # if tasks have been started

            for looptask in self._looping_tasks:

                self.info('canceling task %s...', looptask.name)

                looptask.cancel()

                looptask.join()

                self.info('task %s finished', looptask.name)

        for thread in self._running_threads:

            self.info('waiting thread %s...', thread.name)

            thread.join()

            self.info('thread %s finished', thread.name)

        self.hm.call_hooks('server_shutdown', repo=self)

        self.close_sessions()

        while not self._available_pools.empty():

            pool = self._available_pools.get_nowait()

            try:

                pool.close(True)

            except:

                self.exception('error while closing %s' % pool)

                continue

        if self.pyro_registered:

            pyro_unregister(self.config)

        hits, misses = self.querier.cache_hit, self.querier.cache_miss

        try:

            self.info('rqlt st cache hit/miss: %s/%s (%s%% hits)', hits, misses,

                      (hits * 100) / (hits + misses))

            hits, misses = self.system_source.cache_hit, self.system_source.cache_miss

            self.info('sql cache hit/miss: %s/%s (%s%% hits)', hits, misses,

                      (hits * 100) / (hits + misses))

            nocache  = self.system_source.no_cache

            self.info('sql cache usage: %s/%s (%s%%)', hits+ misses, nocache,