# copyright 2003-2012 LOGILAB S.A. (Paris, FRANCE), all rights reserved.

# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr

#

# This file is part of CubicWeb.

#

# CubicWeb is free software: you can redistribute it and/or modify it under the

# terms of the GNU Lesser General Public License as published by the Free

# Software Foundation, either version 2.1 of the License, or (at your option)

# any later version.

#

# CubicWeb is distributed in the hope that it will be useful, but WITHOUT

# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS

# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more

# details.

#

# You should have received a copy of the GNU Lesser General Public License along

# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.

"""Helper classes to execute RQL queries on a set of sources, performing

security checking and data aggregation.

"""

from __future__ import with_statement

__docformat__ = "restructuredtext en"

from itertools import repeat

from logilab.common.compat import any

from rql import RQLSyntaxError, CoercionError

from rql.stmts import Union, Select

from rql.nodes import ETYPE_PYOBJ_MAP, etype_from_pyobj

from rql.nodes import (Relation, VariableRef, Constant, SubQuery, Function,

                       Exists, Not)

from yams import BASE_TYPES

from cubicweb import ValidationError, Unauthorized, QueryError, UnknownEid

from cubicweb import Binary, server, typed_eid

from cubicweb.rset import ResultSet

from cubicweb.utils import QueryCache, RepeatList

from cubicweb.server.utils import cleanup_solutions

from cubicweb.server.rqlannotation import SQLGenAnnotator, set_qdata

from cubicweb.server.ssplanner import READ_ONLY_RTYPES, add_types_restriction

from cubicweb.server.edition import EditedEntity

ETYPE_PYOBJ_MAP[Binary] = 'Bytes'

def empty_rset(rql, args, rqlst=None):

    """build an empty result set object"""

    return ResultSet([], rql, args, rqlst=rqlst)

def update_varmap(varmap, selected, table):

    """return a sql schema to store RQL query result"""

    for i, term in enumerate(selected):

        key = term.as_string()

        value = '%s.C%s' % (table, i)

        if varmap.get(key, value) != value:

            raise Exception('variable name conflict on %s: got %s / %s'

                            % (key, value, varmap))

        varmap[key] = value

# permission utilities ########################################################

def check_no_password_selected(rqlst):

    """check that Password entities are not selected"""

    for solution in rqlst.solutions:

        if 'Password' in solution.itervalues():

            raise Unauthorized('Password selection is not allowed')

def term_etype(session, term, solution, args):

    """return the entity type for the given term (a VariableRef or a Constant

    node)

    """

    try:

        return solution[term.name]

    except AttributeError:

        return session.describe(term.eval(args))[0]

def check_read_access(session, rqlst, solution, args):

    """check that the given user has credentials to access data read the

    query

    return a dict defining necessary local checks (due to use of rql expression

    in the schema), keys are variable names and values associated rql expression

    for the associated variable with the given solution

    """

    # use `term_etype` since we've to deal with rewritten constants here,

    # when used as an external source by another repository.

    # XXX what about local read security w/ those rewritten constants...

    schema = session.repo.schema

    if rqlst.where is not None:

        for rel in rqlst.where.iget_nodes(Relation):

            # XXX has_text may have specific perm ?

            if rel.r_type in READ_ONLY_RTYPES:

                continue

            rschema = schema.rschema(rel.r_type)

            if rschema.final:

                eschema = schema.eschema(term_etype(session, rel.children[0],

                                                    solution, args))

                rdef = eschema.rdef(rschema)

            else:

                rdef = rschema.rdef(term_etype(session, rel.children[0],

                                               solution, args),

                                    term_etype(session, rel.children[1].children[0],

                                               solution, args))

            if not session.user.matching_groups(rdef.get_groups('read')):

                # XXX rqlexpr not allowed

                raise Unauthorized('read', rel.r_type)

    localchecks = {}

    # iterate on defined_vars and not on solutions to ignore column aliases

    for varname in rqlst.defined_vars:

        eschema = schema.eschema(solution[varname])

        if eschema.final:

            continue

        if not session.user.matching_groups(eschema.get_groups('read')):

            erqlexprs = eschema.get_rqlexprs('read')

            if not erqlexprs:

                ex = Unauthorized('read', solution[varname])

                ex.var = varname

                raise ex

            # don't insert security on variable only referenced by 'NOT X relation Y' or

            # 'NOT EXISTS(X relation Y)'

            varinfo = rqlst.defined_vars[varname].stinfo

            if varinfo['selected'] or (

                len([r for r in varinfo['relations']

                     if (not schema.rschema(r.r_type).final

                         and ((isinstance(r.parent, Exists) and r.parent.neged(strict=True))

                              or isinstance(r.parent, Not)))])

                != len(varinfo['relations'])):

                localchecks[varname] = erqlexprs

    return localchecks

def add_noinvariant(noinvariant, restricted, select, nbtrees):

    # a variable can actually be invariant if it has not been restricted for

    # security reason or if security assertion hasn't modified the possible

    # solutions for the query

    if nbtrees != 1:

        for vname in restricted:

            try:

                noinvariant.add(select.defined_vars[vname])

            except KeyError:

                # this is an alias

                continue

    else:

        for vname in restricted:

            try:

                var = select.defined_vars[vname]

            except KeyError:

                # this is an alias

                continue

            if len(var.stinfo['possibletypes']) != 1:

                noinvariant.add(var)

def _expand_selection(terms, selected, aliases, select, newselect):

    for term in terms:

        for vref in term.iget_nodes(VariableRef):

            if not vref.name in selected:

                select.append_selected(vref)

                colalias = newselect.get_variable(vref.name, len(aliases))

                aliases.append(VariableRef(colalias))

                selected.add(vref.name)

# Plans #######################################################################

class ExecutionPlan(object):

    """the execution model of a rql query, composed of querier steps"""

    def __init__(self, querier, rqlst, args, session):

        # original rql syntax tree

        self.rqlst = rqlst

        self.args = args or {}

        # session executing the query

        self.session = session

        # quick reference to the system source

        self.syssource = session.cnxset.source('system')

        # execution steps

        self.steps = []

        # index of temporary tables created during execution

        self.temp_tables = {}

        # various resource accesors

        self.querier = querier

        self.schema = querier.schema

        self.sqlannotate = querier.sqlgen_annotate

        self.rqlhelper = session.vreg.rqlhelper

    def annotate_rqlst(self):

        if not self.rqlst.annotated:

            self.rqlhelper.annotate(self.rqlst)

    def add_step(self, step):

        """add a step to the plan"""

        self.steps.append(step)

    def clean(self):

        """remove temporary tables"""

        self.syssource.clean_temp_data(self.session, self.temp_tables)

    def sqlexec(self, sql, args=None):

        return self.syssource.sqlexec(self.session, sql, args)

    def execute(self):

        """execute a plan and return resulting rows"""

        try:

            for step in self.steps:

                result = step.execute()

            # the latest executed step contains the full query result

            return result

        finally:

            self.clean()

    def make_temp_table_name(self, table):

        """

        return a temp table name according to db backend

        """

        return self.syssource.make_temp_table_name(table)

    def init_temp_table(self, table, selected, sol):

        """initialize sql schema and variable map for a temporary table which

        will be used to store result for the given rqlst

        """

        try:

            outputmap, sqlschema, _ = self.temp_tables[table]

            update_varmap(outputmap, selected, table)

        except KeyError:

            sqlschema, outputmap = self.syssource.temp_table_def(selected, sol,

                                                                 table)

            self.temp_tables[table] = [outputmap, sqlschema, False]

        return outputmap

    def create_temp_table(self, table):

        """create a temporary table to store result for the given rqlst"""

        if not self.temp_tables[table][-1]:

            sqlschema = self.temp_tables[table][1]

            self.syssource.create_temp_table(self.session, table, sqlschema)

            self.temp_tables[table][-1] = True

    def preprocess(self, union, security=True):

        """insert security when necessary then annotate rql st for sql generation

        return rqlst to actually execute

        """

        cached = None

        if security and self.session.read_security:

            # ensure security is turned of when security is inserted,

            # else we may loop for ever...

            if self.session.transaction_data.get('security-rqlst-cache'):

                key = self.cache_key

            else:

                key = None

            if key is not None and key in self.session.transaction_data:

                cachedunion, args = self.session.transaction_data[key]

                union.children[:] = []

                for select in cachedunion.children:

                    union.append(select)

                union.has_text_query = cachedunion.has_text_query

                args.update(self.args)

                self.args = args

                cached = True

            else:

                noinvariant = set()

                with self.session.security_enabled(read=False):

                    self._insert_security(union, noinvariant)

                if key is not None:

                    self.session.transaction_data[key] = (union, self.args)

        else:

            noinvariant = ()

        if cached is None:

            self.rqlhelper.simplify(union)

            self.sqlannotate(union)

            set_qdata(self.schema.rschema, union, noinvariant)

        if union.has_text_query:

            self.cache_key = None

    def _insert_security(self, union, noinvariant):

        for select in union.children[:]:

            for subquery in select.with_:

                self._insert_security(subquery.query, noinvariant)

            localchecks, restricted = self._check_permissions(select)

            if any(localchecks):

                rewrite = self.session.rql_rewriter.rewrite

                nbtrees = len(localchecks)

                myunion = union

                # transform in subquery when len(localchecks)>1 and groups

                if nbtrees > 1 and (select.orderby or select.groupby or

                                    select.having or select.has_aggregat or

                                    select.distinct or

                                    select.limit or select.offset):

                    newselect = Select()

                    # only select variables in subqueries

                    origselection = select.selection

                    select.select_only_variables()

                    select.has_aggregat = False

                    # create subquery first so correct node are used on copy

                    # (eg ColumnAlias instead of Variable)

                    aliases = [VariableRef(newselect.get_variable(vref.name, i))

                               for i, vref in enumerate(select.selection)]

                    selected = set(vref.name for vref in aliases)

                    # now copy original selection and groups

                    for term in origselection:

                        newselect.append_selected(term.copy(newselect))

                    if select.orderby:

                        sortterms = []

                        for sortterm in select.orderby:

                            sortterms.append(sortterm.copy(newselect))

                            for fnode in sortterm.get_nodes(Function):

                                if fnode.name == 'FTIRANK':

                                    # we've to fetch the has_text relation as well

                                    var = fnode.children[0].variable

                                    rel = iter(var.stinfo['ftirels']).next()

                                    assert not rel.ored(), 'unsupported'

                                    newselect.add_restriction(rel.copy(newselect))

                                    # remove relation from the orig select and

                                    # cleanup variable stinfo

                                    rel.parent.remove(rel)

                                    var.stinfo['ftirels'].remove(rel)

                                    var.stinfo['relations'].remove(rel)

                                    # XXX not properly re-annotated after security insertion?

                                    newvar = newselect.get_variable(var.name)

                                    newvar.stinfo.setdefault('ftirels', set()).add(rel)

                                    newvar.stinfo.setdefault('relations', set()).add(rel)

                        newselect.set_orderby(sortterms)

                        _expand_selection(select.orderby, selected, aliases, select, newselect)

                        select.orderby = () # XXX dereference?

                    if select.groupby:

                        newselect.set_groupby([g.copy(newselect) for g in select.groupby])

                        _expand_selection(select.groupby, selected, aliases, select, newselect)

                        select.groupby = () # XXX dereference?

                    if select.having:

                        newselect.set_having([g.copy(newselect) for g in select.having])

                        _expand_selection(select.having, selected, aliases, select, newselect)

                        select.having = () # XXX dereference?

                    if select.limit:

                        newselect.limit = select.limit

                        select.limit = None

                    if select.offset:

                        newselect.offset = select.offset

                        select.offset = 0

                    myunion = Union()

                    newselect.set_with([SubQuery(aliases, myunion)], check=False)

                    newselect.distinct = select.distinct

                    solutions = [sol.copy() for sol in select.solutions]

                    cleanup_solutions(newselect, solutions)

                    newselect.set_possible_types(solutions)

                    # if some solutions doesn't need rewriting, insert original

                    # select as first union subquery

                    if () in localchecks:

                        myunion.append(select)

                    # we're done, replace original select by the new select with

                    # subqueries (more added in the loop below)

                    union.replace(select, newselect)

                elif not () in localchecks:

                    union.remove(select)

                for lcheckdef, lchecksolutions in localchecks.iteritems():

                    if not lcheckdef:

                        continue

                    myrqlst = select.copy(solutions=lchecksolutions)

                    myunion.append(myrqlst)

                    # in-place rewrite + annotation / simplification

                    lcheckdef = [({var: 'X'}, rqlexprs) for var, rqlexprs in lcheckdef]

                    rewrite(myrqlst, lcheckdef, lchecksolutions, self.args)

                    add_noinvariant(noinvariant, restricted, myrqlst, nbtrees)

                if () in localchecks:

                    select.set_possible_types(localchecks[()])

                    add_types_restriction(self.schema, select)

                    add_noinvariant(noinvariant, restricted, select, nbtrees)

                self.rqlhelper.annotate(union)

    def _check_permissions(self, rqlst):

        """return a dict defining "local checks", e.g. RQLExpression defined in

        the schema that should be inserted in the original query

        solutions where a variable has a type which the user can't definitly read

        are removed, else if the user may read it (eg if an rql expression is

        defined for the "read" permission of the related type), the local checks