# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.

# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr

#

# This file is part of CubicWeb.

#

# CubicWeb is free software: you can redistribute it and/or modify it under the

# terms of the GNU Lesser General Public License as published by the Free

# Software Foundation, either version 2.1 of the License, or (at your option)

# any later version.

#

# CubicWeb is distributed in the hope that it will be useful, but WITHOUT

# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS

# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more

# details.

#

# You should have received a copy of the GNU Lesser General Public License along

# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.

"""Adapters for native cubicweb sources.

Notes:

* extid (aka external id, the primary key of an entity in the external source

  from which it comes from) are stored in a varchar column encoded as a base64

  string. This is because it should actually be Bytes but we want an index on

  it for fast querying.

"""

from __future__ import with_statement

__docformat__ = "restructuredtext en"

try:

    from cPickle import loads, dumps

    import cPickle as pickle

except ImportError:

    from pickle import loads, dumps

    import pickle

from threading import Lock

from datetime import datetime

from base64 import b64decode, b64encode

from contextlib import contextmanager

from os.path import abspath, basename

import re

import itertools

import zipfile

import logging

import sys

from logilab.common.compat import any

from logilab.common.decorators import cached, clear_cache

from logilab.common.configuration import Method

from logilab.common.shellutils import getlogin

from logilab.database import get_db_helper, sqlgen

from yams import schema2sql as y2sql

from yams.schema import role_name

from cubicweb import (UnknownEid, AuthenticationError, ValidationError, Binary,

                      UniqueTogetherError, QueryError, UndoTransactionException)

from cubicweb import transaction as tx, server, neg_role

from cubicweb.utils import QueryCache

from cubicweb.schema import VIRTUAL_RTYPES

from cubicweb.cwconfig import CubicWebNoAppConfiguration

from cubicweb.server import hook

from cubicweb.server.utils import crypt_password, eschema_eid

from cubicweb.server.sqlutils import SQL_PREFIX, SQLAdapterMixIn

from cubicweb.server.rqlannotation import set_qdata

from cubicweb.server.hook import CleanupDeletedEidsCacheOp

from cubicweb.server.edition import EditedEntity

from cubicweb.server.sources import AbstractSource, dbg_st_search, dbg_results

from cubicweb.server.sources.rql2sql import SQLGenerator

ATTR_MAP = {}

NONSYSTEM_ETYPES = set()

NONSYSTEM_RELATIONS = set()

class LogCursor(object):

    def __init__(self, cursor):

        self.cu = cursor

    def execute(self, query, args=None):

        """Execute a query.

        it's a function just so that it shows up in profiling

        """

        if server.DEBUG & server.DBG_SQL:

            print 'exec', query, args

        try:

            self.cu.execute(str(query), args)

        except Exception, ex:

            print "sql: %r\n args: %s\ndbms message: %r" % (

                query, args, ex.args[0])

            raise

    def fetchall(self):

        return self.cu.fetchall()

    def fetchone(self):

        return self.cu.fetchone()

def make_schema(selected, solution, table, typemap):

    """return a sql schema to store RQL query result"""

    sql = []

    varmap = {}

    for i, term in enumerate(selected):

        name = 'C%s' % i

        key = term.as_string()

        varmap[key] = '%s.%s' % (table, name)

        ttype = term.get_type(solution)

        try:

            sql.append('%s %s' % (name, typemap[ttype]))

        except KeyError:

            # assert not schema(ttype).final

            sql.append('%s %s' % (name, typemap['Int']))

    return ','.join(sql), varmap

def _modified_sql(table, etypes):

    # XXX protect against sql injection

    if len(etypes) > 1:

        restr = 'type IN (%s)' % ','.join("'%s'" % etype for etype in etypes)

    else:

        restr = "type='%s'" % etypes[0]

    if table == 'entities':

        attr = 'mtime'

    else:

        attr = 'dtime'

    return 'SELECT type, eid FROM %s WHERE %s AND %s > %%(time)s' % (

        table, restr, attr)

def sql_or_clauses(sql, clauses):

    select, restr = sql.split(' WHERE ', 1)

    restrclauses = restr.split(' AND ')

    for clause in clauses:

        restrclauses.remove(clause)

    if restrclauses:

        restr = '%s AND (%s)' % (' AND '.join(restrclauses),

                                 ' OR '.join(clauses))

    else:

        restr = '(%s)' % ' OR '.join(clauses)

    return '%s WHERE %s' % (select, restr)

def rdef_table_column(rdef):

    """return table and column used to store the given relation definition in

    the database

    """

    return (SQL_PREFIX + str(rdef.subject),

            SQL_PREFIX + str(rdef.rtype))

def rdef_physical_info(dbhelper, rdef):

    """return backend type and a boolean flag if NULL values should be allowed

    for a given relation definition

    """

    if rdef.object.final:

        ttype = rdef.object

    else:

        ttype = 'Int' # eid type

    coltype = y2sql.type_from_constraints(dbhelper, ttype,

                                          rdef.constraints, creating=False)

    allownull = rdef.cardinality[0] != '1'

    return coltype, allownull

class _UndoException(Exception):

    """something went wrong during undoing"""

    def __unicode__(self):

        """Called by the unicode builtin; should return a Unicode object

        Type of _UndoException message must be `unicode` by design in CubicWeb.

        """

        assert isinstance(self.args[0], unicode)

        return self.args[0]

def _undo_check_relation_target(tentity, rdef, role):

    """check linked entity has not been redirected for this relation"""

    card = rdef.role_cardinality(role)

    if card in '?1' and tentity.related(rdef.rtype, role):

        raise _UndoException(tentity._cw._(

            "Can't restore %(role)s relation %(rtype)s to entity %(eid)s which "

            "is already linked using this relation.")

                            % {'role': neg_role(role),

                               'rtype': rdef.rtype,

                               'eid': tentity.eid})

def _undo_rel_info(session, subj, rtype, obj):

    entities = []

    for role, eid in (('subject', subj), ('object', obj)):

        try:

            entities.append(session.entity_from_eid(eid))

        except UnknownEid:

            raise _UndoException(session._(

                "Can't restore relation %(rtype)s, %(role)s entity %(eid)s"

                " doesn't exist anymore.")

                                % {'role': session._(role),

                                   'rtype': session._(rtype),

                                   'eid': eid})

    sentity, oentity = entities

    try:

        rschema = session.vreg.schema.rschema(rtype)

        rdef = rschema.rdefs[(sentity.__regid__, oentity.__regid__)]

    except KeyError:

        raise _UndoException(session._(

            "Can't restore relation %(rtype)s between %(subj)s and "

            "%(obj)s, that relation does not exists anymore in the "

            "schema.")

                            % {'rtype': session._(rtype),

                               'subj': subj,

                               'obj': obj})

    return sentity, oentity, rdef

def _undo_has_later_transaction(session, eid):

    return session.system_sql('''\

SELECT T.tx_uuid FROM transactions AS TREF, transactions AS T

WHERE TREF.tx_uuid='%(txuuid)s' AND T.tx_uuid!='%(txuuid)s'

AND T.tx_time>=TREF.tx_time

AND (EXISTS(SELECT 1 FROM tx_entity_actions AS TEA

            WHERE TEA.tx_uuid=T.tx_uuid AND TEA.eid=%(eid)s)

     OR EXISTS(SELECT 1 FROM tx_relation_actions as TRA

               WHERE TRA.tx_uuid=T.tx_uuid AND (

                   TRA.eid_from=%(eid)s OR TRA.eid_to=%(eid)s))

     )''' % {'txuuid': session.transaction_data['undoing_uuid'],

             'eid': eid}).fetchone()

class NativeSQLSource(SQLAdapterMixIn, AbstractSource):

    """adapter for source using the native cubicweb schema (see below)

    """

    sqlgen_class = SQLGenerator

    options = (

        ('db-driver',

         {'type' : 'string',

          'default': 'postgres',

          # XXX use choice type

          'help': 'database driver (postgres, mysql, sqlite, sqlserver2005)',

          'group': 'native-source', 'level': 0,

          }),

        ('db-host',

         {'type' : 'string',

          'default': '',

          'help': 'database host',

          'group': 'native-source', 'level': 1,

          }),

        ('db-port',

         {'type' : 'string',

          'default': '',

          'help': 'database port',

          'group': 'native-source', 'level': 1,

          }),

        ('db-name',

         {'type' : 'string',

          'default': Method('default_instance_id'),

          'help': 'database name',

          'group': 'native-source', 'level': 0,

          }),

        ('db-user',

         {'type' : 'string',

          'default': CubicWebNoAppConfiguration.mode == 'user' and getlogin() or 'cubicweb',

          'help': 'database user',

          'group': 'native-source', 'level': 0,

          }),

        ('db-password',

         {'type' : 'password',

          'default': '',

          'help': 'database password',

          'group': 'native-source', 'level': 0,

          }),

        ('db-encoding',

         {'type' : 'string',

          'default': 'utf8',

          'help': 'database encoding',

          'group': 'native-source', 'level': 1,

          }),

        ('db-extra-arguments',

         {'type' : 'string',

          'default': '',

          'help': 'set to "Trusted_Connection" if you are using SQLServer and '

                  'want trusted authentication for the database connection',

          'group': 'native-source', 'level': 2,

          }),

    )

    def __init__(self, repo, source_config, *args, **kwargs):

        SQLAdapterMixIn.__init__(self, source_config)

        self.authentifiers = [LoginPasswordAuthentifier(self)]

        if repo.config['allow-email-login']:

            self.authentifiers.insert(0, EmailPasswordAuthentifier(self))

        AbstractSource.__init__(self, repo, source_config, *args, **kwargs)

        # sql generator

        self._rql_sqlgen = self.sqlgen_class(self.schema, self.dbhelper,

                                             ATTR_MAP.copy())

        # full text index helper

        self.do_fti = not repo.config['delay-full-text-indexation']

        # sql queries cache

        self._cache = QueryCache(repo.config['rql-cache-size'])

        self._temp_table_data = {}

        # we need a lock to protect eid attribution function (XXX, really?

        # explain)

        self._eid_cnx_lock = Lock()

        self._eid_creation_cnx = None

        # (etype, attr) / storage mapping

        self._storages = {}

        # entity types that may be used by other multi-sources instances

        self.multisources_etypes = set(repo.config['multi-sources-etypes'])

        # XXX no_sqlite_wrap trick since we've a sqlite locking pb when

        # running unittest_multisources with the wrapping below

        if self.dbdriver == 'sqlite' and \

               not getattr(repo.config, 'no_sqlite_wrap', False):

            from cubicweb.server.sources.extlite import ConnectionWrapper

            self.dbhelper.dbname = abspath(self.dbhelper.dbname)

            self.get_connection = lambda: ConnectionWrapper(self)

            self.check_connection = lambda cnx: cnx

            def cnxset_freed(cnx):

                cnx.close()

            self.cnxset_freed = cnxset_freed

        if self.dbdriver == 'sqlite':

            self._create_eid = None

            self.create_eid = self._create_eid_sqlite

        self.binary_to_str = self.dbhelper.dbapi_module.binary_to_str

    @property

    def _sqlcnx(self):

        # XXX: sqlite connections can only be used in the same thread, so

        #      create a new one each time necessary. If it appears to be time

        #      consuming, find another way

        return SQLAdapterMixIn.get_connection(self)

    def check_config(self, source_entity):

        """check configuration of source entity"""

        if source_entity.host_config:

            msg = source_entity._cw._('the system source has its configuration '

                                      'stored on the file-system')

            raise ValidationError(source_entity.eid, {role_name('config', 'subject'): msg})

    def add_authentifier(self, authentifier):

        self.authentifiers.append(authentifier)

        authentifier.source = self

        authentifier.set_schema(self.schema)

    def reset_caches(self):

        """method called during test to reset potential source caches"""

        self._cache = QueryCache(self.repo.config['rql-cache-size'])

    def clear_eid_cache(self, eid, etype):

        """clear potential caches for the given eid"""

        self._cache.pop('Any X WHERE X eid %s, X is %s' % (eid, etype), None)

        self._cache.pop('Any X WHERE X eid %s' % eid, None)

        self._cache.pop('Any %s' % eid, None)

    def sqlexec(self, session, sql, args=None):

        """execute the query and return its result"""

        return self.process_result(self.doexec(session, sql, args))

    def init_creating(self, cnxset=None):

        # check full text index availibility

        if self.do_fti:

            if cnxset is None:

                _cnxset = self.repo._get_cnxset()

                _cnxset.cnxset_set()

            else:

                _cnxset = cnxset

            if not self.dbhelper.has_fti_table(_cnxset['system']):

                if not self.repo.config.creating:

                    self.critical('no text index table')

                self.do_fti = False

            if cnxset is None:

                _cnxset.cnxset_freed()

                self.repo._free_cnxset(_cnxset)

    def backup(self, backupfile, confirm, format='native'):

        """method called to create a backup of the source's data"""

        if format == 'portable':

            self.repo.fill_schema()

            self.set_schema(self.repo.schema)

            helper = DatabaseIndependentBackupRestore(self)