# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.

# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr

#

# This file is part of CubicWeb.

#

# CubicWeb is free software: you can redistribute it and/or modify it under the

# terms of the GNU Lesser General Public License as published by the Free

# Software Foundation, either version 2.1 of the License, or (at your option)

# any later version.

#

# CubicWeb is distributed in the hope that it will be useful, but WITHOUT

# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS

# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more

# details.

#

# You should have received a copy of the GNU Lesser General Public License along

# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.

"""Adapters for native cubicweb sources.

Notes:

* extid (aka external id, the primary key of an entity in the external source

  from which it comes from) are stored in a varchar column encoded as a base64

  string. This is because it should actually be Bytes but we want an index on

  it for fast querying.

"""

__docformat__ = "restructuredtext en"

try:

    from cPickle import loads, dumps

    import cPickle as pickle

except ImportError:

    from pickle import loads, dumps

    import pickle

from threading import Lock

from datetime import datetime

from base64 import b64decode, b64encode

from contextlib import contextmanager

from os.path import abspath, basename

import re

import itertools

import zipfile

import logging

import sys

from logilab.common.compat import any

from logilab.common.decorators import cached, clear_cache

from logilab.common.configuration import Method

from logilab.common.shellutils import getlogin

from logilab.database import get_db_helper, sqlgen

from yams import schema2sql as y2sql

from yams.schema import role_name

from cubicweb import (UnknownEid, AuthenticationError, ValidationError, Binary,

                      UniqueTogetherError, QueryError, UndoTransactionException)

from cubicweb import transaction as tx, server, neg_role

from cubicweb.utils import QueryCache

from cubicweb.schema import VIRTUAL_RTYPES

from cubicweb.cwconfig import CubicWebNoAppConfiguration

from cubicweb.server import hook

from cubicweb.server.utils import crypt_password, eschema_eid, verify_and_update

from cubicweb.server.sqlutils import SQL_PREFIX, SQLAdapterMixIn, SqliteCnxLoggingWrapper

from cubicweb.server.rqlannotation import set_qdata

from cubicweb.server.hook import CleanupDeletedEidsCacheOp

from cubicweb.server.edition import EditedEntity

from cubicweb.server.sources import AbstractSource, dbg_st_search, dbg_results

from cubicweb.server.sources.rql2sql import SQLGenerator

ATTR_MAP = {}

NONSYSTEM_ETYPES = set()

NONSYSTEM_RELATIONS = set()

class LogCursor(object):

    def __init__(self, cursor):

        self.cu = cursor

    def execute(self, query, args=None):

        """Execute a query.

        it's a function just so that it shows up in profiling

        """

        if server.DEBUG & server.DBG_SQL:

            print 'exec', query, args

        try:

            self.cu.execute(str(query), args)

        except Exception as ex:

            print "sql: %r\n args: %s\ndbms message: %r" % (

                query, args, ex.args[0])

            raise

    def fetchall(self):

        return self.cu.fetchall()

    def fetchone(self):

        return self.cu.fetchone()

def make_schema(selected, solution, table, typemap):

    """return a sql schema to store RQL query result"""

    sql = []

    varmap = {}

    for i, term in enumerate(selected):

        name = 'C%s' % i

        key = term.as_string()

        varmap[key] = '%s.%s' % (table, name)

        ttype = term.get_type(solution)

        try:

            sql.append('%s %s' % (name, typemap[ttype]))

        except KeyError:

            # assert not schema(ttype).final

            sql.append('%s %s' % (name, typemap['Int']))

    return ','.join(sql), varmap

def sql_or_clauses(sql, clauses):

    select, restr = sql.split(' WHERE ', 1)

    restrclauses = restr.split(' AND ')

    for clause in clauses:

        restrclauses.remove(clause)

    if restrclauses:

        restr = '%s AND (%s)' % (' AND '.join(restrclauses),

                                 ' OR '.join(clauses))

    else:

        restr = '(%s)' % ' OR '.join(clauses)

    return '%s WHERE %s' % (select, restr)

def rdef_table_column(rdef):

    """return table and column used to store the given relation definition in

    the database

    """

    return (SQL_PREFIX + str(rdef.subject),

            SQL_PREFIX + str(rdef.rtype))

def rdef_physical_info(dbhelper, rdef):

    """return backend type and a boolean flag if NULL values should be allowed

    for a given relation definition

    """

    if rdef.object.final:

        ttype = rdef.object

    else:

        ttype = 'Int' # eid type

    coltype = y2sql.type_from_constraints(dbhelper, ttype,

                                          rdef.constraints, creating=False)

    allownull = rdef.cardinality[0] != '1'

    return coltype, allownull

class _UndoException(Exception):

    """something went wrong during undoing"""

    def __unicode__(self):

        """Called by the unicode builtin; should return a Unicode object

        Type of _UndoException message must be `unicode` by design in CubicWeb.

        """

        assert isinstance(self.args[0], unicode)

        return self.args[0]

def _undo_check_relation_target(tentity, rdef, role):

    """check linked entity has not been redirected for this relation"""

    card = rdef.role_cardinality(role)

    if card in '?1' and tentity.related(rdef.rtype, role):

        raise _UndoException(tentity._cw._(

            "Can't restore %(role)s relation %(rtype)s to entity %(eid)s which "

            "is already linked using this relation.")

                            % {'role': neg_role(role),

                               'rtype': rdef.rtype,

                               'eid': tentity.eid})

def _undo_rel_info(session, subj, rtype, obj):

    entities = []

    for role, eid in (('subject', subj), ('object', obj)):

        try:

            entities.append(session.entity_from_eid(eid))

        except UnknownEid:

            raise _UndoException(session._(

                "Can't restore relation %(rtype)s, %(role)s entity %(eid)s"

                " doesn't exist anymore.")

                                % {'role': session._(role),

                                   'rtype': session._(rtype),

                                   'eid': eid})

    sentity, oentity = entities

    try:

        rschema = session.vreg.schema.rschema(rtype)

        rdef = rschema.rdefs[(sentity.cw_etype, oentity.cw_etype)]

    except KeyError:

        raise _UndoException(session._(

            "Can't restore relation %(rtype)s between %(subj)s and "

            "%(obj)s, that relation does not exists anymore in the "

            "schema.")

                            % {'rtype': session._(rtype),

                               'subj': subj,

                               'obj': obj})

    return sentity, oentity, rdef

def _undo_has_later_transaction(session, eid):

    return session.system_sql('''\

SELECT T.tx_uuid FROM transactions AS TREF, transactions AS T

WHERE TREF.tx_uuid='%(txuuid)s' AND T.tx_uuid!='%(txuuid)s'

AND T.tx_time>=TREF.tx_time

AND (EXISTS(SELECT 1 FROM tx_entity_actions AS TEA

            WHERE TEA.tx_uuid=T.tx_uuid AND TEA.eid=%(eid)s)

     OR EXISTS(SELECT 1 FROM tx_relation_actions as TRA

               WHERE TRA.tx_uuid=T.tx_uuid AND (

                   TRA.eid_from=%(eid)s OR TRA.eid_to=%(eid)s))

     )''' % {'txuuid': session.transaction_data['undoing_uuid'],

             'eid': eid}).fetchone()

class NativeSQLSource(SQLAdapterMixIn, AbstractSource):

    """adapter for source using the native cubicweb schema (see below)

    """

    sqlgen_class = SQLGenerator

    options = (

        ('db-driver',

         {'type' : 'string',

          'default': 'postgres',

          # XXX use choice type

          'help': 'database driver (postgres, mysql, sqlite, sqlserver2005)',

          'group': 'native-source', 'level': 0,

          }),

        ('db-host',

         {'type' : 'string',

          'default': '',

          'help': 'database host',

          'group': 'native-source', 'level': 1,

          }),

        ('db-port',

         {'type' : 'string',

          'default': '',

          'help': 'database port',

          'group': 'native-source', 'level': 1,

          }),

        ('db-name',

         {'type' : 'string',

          'default': Method('default_instance_id'),

          'help': 'database name',

          'group': 'native-source', 'level': 0,

          }),

        ('db-user',

         {'type' : 'string',

          'default': CubicWebNoAppConfiguration.mode == 'user' and getlogin() or 'cubicweb',

          'help': 'database user',

          'group': 'native-source', 'level': 0,

          }),

        ('db-password',

         {'type' : 'password',

          'default': '',

          'help': 'database password',

          'group': 'native-source', 'level': 0,

          }),

        ('db-encoding',

         {'type' : 'string',

          'default': 'utf8',

          'help': 'database encoding',

          'group': 'native-source', 'level': 1,

          }),

        ('db-extra-arguments',

         {'type' : 'string',

          'default': '',

          'help': 'set to "Trusted_Connection" if you are using SQLServer and '

                  'want trusted authentication for the database connection',

          'group': 'native-source', 'level': 2,

          }),

    )

    def __init__(self, repo, source_config, *args, **kwargs):

        SQLAdapterMixIn.__init__(self, source_config)

        self.authentifiers = [LoginPasswordAuthentifier(self)]

        if repo.config['allow-email-login']:

            self.authentifiers.insert(0, EmailPasswordAuthentifier(self))

        AbstractSource.__init__(self, repo, source_config, *args, **kwargs)

        # sql generator

        self._rql_sqlgen = self.sqlgen_class(self.schema, self.dbhelper,

                                             ATTR_MAP.copy())

        # full text index helper

        self.do_fti = not repo.config['delay-full-text-indexation']

        # sql queries cache

        self._cache = QueryCache(repo.config['rql-cache-size'])

        self._temp_table_data = {}

        # we need a lock to protect eid attribution function (XXX, really?

        # explain)

        self._eid_cnx_lock = Lock()

        self._eid_creation_cnx = None

        # (etype, attr) / storage mapping

        self._storages = {}

        # XXX no_sqlite_wrap trick since we've a sqlite locking pb when

        # running unittest_multisources with the wrapping below

        if self.dbdriver == 'sqlite' and \

               not getattr(repo.config, 'no_sqlite_wrap', False):

            self.dbhelper.dbname = abspath(self.dbhelper.dbname)

            self.get_connection = lambda: SqliteCnxLoggingWrapper(self)

            self.check_connection = lambda cnx: cnx

            def cnxset_freed(cnx):

                cnx.close()

            self.cnxset_freed = cnxset_freed

        if self.dbdriver == 'sqlite':

            self._create_eid = None

            self.create_eid = self._create_eid_sqlite

        self.binary_to_str = self.dbhelper.dbapi_module.binary_to_str

    def check_config(self, source_entity):

        """check configuration of source entity"""

        if source_entity.host_config:

            msg = source_entity._cw._('the system source has its configuration '

                                      'stored on the file-system')

            raise ValidationError(source_entity.eid, {role_name('config', 'subject'): msg})

    def add_authentifier(self, authentifier):

        self.authentifiers.append(authentifier)

        authentifier.source = self

        authentifier.set_schema(self.schema)

    def reset_caches(self):

        """method called during test to reset potential source caches"""

        self._cache = QueryCache(self.repo.config['rql-cache-size'])

    def clear_eid_cache(self, eid, etype):

        """clear potential caches for the given eid"""

        self._cache.pop('Any X WHERE X eid %s, X is %s' % (eid, etype), None)

        self._cache.pop('Any X WHERE X eid %s' % eid, None)

        self._cache.pop('Any %s' % eid, None)

    def sqlexec(self, session, sql, args=None):

        """execute the query and return its result"""

        return self.process_result(self.doexec(session, sql, args))

    def init_creating(self, cnxset=None):

        # check full text index availibility

        if self.do_fti:

            if cnxset is None:

                _cnxset = self.repo._get_cnxset()

                _cnxset.cnxset_set()

            else:

                _cnxset = cnxset

            if not self.dbhelper.has_fti_table(_cnxset['system']):

                if not self.repo.config.creating:

                    self.critical('no text index table')

                self.do_fti = False

            if cnxset is None:

                _cnxset.cnxset_freed()

                self.repo._free_cnxset(_cnxset)

    def backup(self, backupfile, confirm, format='native'):

        """method called to create a backup of the source's data"""

        if format == 'portable':

            # ensure the schema is the one stored in the database: if repository

            # started in quick_start mode, the file system's one has been loaded

            # so force reload

            if self.repo.config.quick_start:

                self.repo.set_schema(self.repo.deserialize_schema(),

                                     resetvreg=False)

            helper = DatabaseIndependentBackupRestore(self)

            self.close_source_connections()

            try:

                helper.backup(backupfile)

            finally:

                self.open_source_connections()

        elif format == 'native':

            self.close_source_connections()

            try:

                self.backup_to_file(backupfile, confirm)

            finally:

                self.open_source_connections()

        else:

            raise ValueError('Unknown format %r' % format)

    def restore(self, backupfile, confirm, drop, format='native'):

        """method called to restore a backup of source's data"""

        if self.repo.config.init_cnxset_pool:

            self.close_source_connections()

        try:

            if format == 'portable':