# copyright 2015 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
# contact http://www.logilab.fr -- mailto:contact@logilab.fr
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 2.1 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
"""This module contains tools to programmatically import external data into CubicWeb. It's designed
on top of the store concept to leverage possibility of code sharing accross various data import
needs.

The following classes are defined:

* :class:`ExtEntity`: some intermediate representation of data to import, using external identifier
  but no eid,

* :class:`ExtEntitiesImporter`: class responsible for turning ExtEntity's extid to eid, and create
  or update CubicWeb entities accordingly (using a Store).

What is left to do is to write a class or a function that will yield external entities from some
data source (eg RDF, CSV) which will be case dependant (the *generator*).  You may then plug
arbitrary filters into the external entities stream between the generator and the importer, allowing
to have some generic generators whose generated content is rafined by specific filters.

.. code-block:: python

    ext_entities = fetch(<source>) # function yielding external entities
    log = SimpleImportLog('<source file/url/whatever>')
    importer = ExtEntitiesImporter(cnx, store, import_log=log)
    importer.import_entities(ext_entities)

Here are the two classes that you'll have to deal with, and maybe to override:

.. autoclass:: cubicweb.dataimport.importer.ExtEntitiesImporter
.. autoclass:: cubicweb.dataimport.importer.ExtEntity
"""

from collections import defaultdict
import logging

from logilab.mtconverter import xml_escape


def cwuri2eid(cnx, etypes, source_eid=None):
    """Return a dictionary mapping cwuri to eid for entities of the given entity types and / or
    source.
    """
    assert source_eid or etypes, 'no entity types nor source specified'
    rql = 'Any U, X WHERE X cwuri U'
    args = {}
    if len(etypes) == 1:
        rql += ', X is %s' % etypes[0]
    elif etypes:
        rql += ', X is IN (%s)' % ','.join(etypes)
    if source_eid is not None:
        rql += ', X cw_source S, S eid %(s)s'
        args['s'] = source_eid
    return dict(cnx.execute(rql, args))


class RelationMapping(object):
    """Read-only mapping from relation type to set of related (subject, object) eids.

    If `source` is specified, only returns relations implying entities from
    this source.
    """

    def __init__(self, cnx, source=None):
        self.cnx = cnx
        self._rql_template = 'Any S,O WHERE S {} O'
        self._kwargs = {}
        if source is not None:
            self._rql_template += ', S cw_source SO, O cw_source SO, SO eid %(s)s'
            self._kwargs['s'] = source.eid

    def __getitem__(self, rtype):
        """Return a set of (subject, object) eids already related by `rtype`"""
        rql = self._rql_template.format(rtype)
        return set(tuple(x) for x in self.cnx.execute(rql, self._kwargs))


class ExtEntity(object):
    """Transitional representation of an entity for use in data importer.

    An external entity has the following properties:

    * ``extid`` (external id), an identifier for the ext entity,
    * ``etype`` (entity type), a string which must be the name of one entity type in the schema
      (eg. ``'Person'``, ``'Animal'``, ...),
    * ``values``, a dictionary whose keys are attribute or relation names from the schema (eg.
      ``'first_name'``, ``'friend'``), and whose values are *sets*

    For instance:

    ..code-block::python

        ext_entity.extid = 'http://example.org/person/debby'
        ext_entity.etype = 'Person'
        ext_entity.values = {'first_name': set([u"Deborah", u"Debby"]),
                            'friend': set(['http://example.org/person/john'])}

    """

    def __init__(self, etype, extid, values=None):
        self.etype = etype
        self.extid = extid
        if values is None:
            values = {}
        self.values = values
        self._schema = None

    def __repr__(self):
        return '<%s %s %s>' % (self.etype, self.extid, self.values)

    def iter_rdefs(self):
        """Yield (key, rtype, role) defined in `.values` dict, with:

        * `key` is the original key in `.values` (i.e. the relation type or a 2-uple (relation type,
          role))

        * `rtype` is a yams relation type, expected to be found in the schema (attribute or
          relation)

        * `role` is the role of the entity in the relation, 'subject' or 'object'

        Iteration is done on a copy of the keys so values may be inserted/deleted during it.
        """
        for key in list(self.values):
            if isinstance(key, tuple):
                rtype, role = key
                assert role in ('subject', 'object'), key
                yield key, rtype, role
            else:
                yield key, key, 'subject'

    def prepare(self, schema):
        """Prepare an external entity for later insertion:

        * ensure attributes and inlined relations have a single value
        * turn set([value]) into value and remove key associated to empty set
        * remove non inlined relations and return them as a [(e1key, relation, e2key)] list

        Return a list of non inlined relations that may be inserted later, each relations defined by
        a 3-tuple (subject extid, relation type, object extid).

        Take care the importer may call this method several times.
        """
        assert self._schema is None, 'prepare() has already been called for %s' % self
        self._schema = schema
        eschema = schema.eschema(self.etype)
        deferred = []
        entity_dict = self.values
        for key, rtype, role in self.iter_rdefs():
            rschema = schema.rschema(rtype)
            if rschema.final or (rschema.inlined and role == 'subject'):
                assert len(entity_dict[key]) <= 1, \
                    "more than one value for %s: %s (%s)" % (rtype, entity_dict[key], self.extid)
                if entity_dict[key]:
                    entity_dict[rtype] = entity_dict[key].pop()
                    if key != rtype:
                        del entity_dict[key]
                    if (rschema.final and eschema.has_metadata(rtype, 'format')
                            and not rtype + '_format' in entity_dict):
                        entity_dict[rtype + '_format'] = u'text/plain'
                else:
                    del entity_dict[key]
            else:
                for target_extid in entity_dict.pop(key):
                    if role == 'subject':
                        deferred.append((self.extid, rtype, target_extid))
                    else:
                        deferred.append((target_extid, rtype, self.extid))
        return deferred

    def is_ready(self, extid2eid):
        """Return True if the ext entity is ready, i.e. has all the URIs used in inlined relations
        currently existing.
        """
        assert self._schema, 'prepare() method should be called first on %s' % self
        # as .prepare has been called, we know that .values only contains subject relation *type* as
        # key (no more (rtype, role) tuple)
        schema = self._schema
        entity_dict = self.values
        for rtype in entity_dict:
            rschema = schema.rschema(rtype)
            if not rschema.final:
                # .prepare() should drop other cases from the entity dict
                assert rschema.inlined
                if not entity_dict[rtype] in extid2eid:
                    return False
        # entity is ready, replace all relation's extid by eids
        for rtype in entity_dict:
            rschema = schema.rschema(rtype)
            if rschema.inlined:
                entity_dict[rtype] = extid2eid[entity_dict[rtype]]
        return True


class ExtEntitiesImporter(object):
    """This class is responsible for importing externals entities, that is instances of
    :class:`ExtEntity`, into CubicWeb entities.

    Parameters:

    * `schema`: the CubicWeb's instance schema

    * `store`: a CubicWeb `Store`

    * `extid2eid`: optional {extid: eid} dictionary giving information on existing entities. It
    will be completed during import. You may want to use :func:`cwuri2eid` to build it.

    * `existing_relation`: optional {rtype: set((subj eid, obj eid))} mapping giving information on
    existing relations of a given type. You may want to use :class:`RelationMapping` to build it.

    * `etypes_order_hint`: optional ordered iterable on entity types, giving an hint on the order in
      which they should be attempted to be imported

    * `import_log`: optional object implementing the :class:`SimpleImportLog` interface to record
      events occuring during the import

    * `raise_on_error`: optional boolean flag - default to false, indicating whether errors should
      be raised or logged. You usually want them to be raised during test but to be logged in
      production.
    """

    def __init__(self, schema, store, extid2eid=None, existing_relations=None,
                 etypes_order_hint=(), import_log=None, raise_on_error=False):
        self.schema = schema
        self.store = store
        self.extid2eid = extid2eid if extid2eid is not None else {}
        self.existing_relations = (existing_relations if existing_relations is not None
                                   else defaultdict(set))
        self.etypes_order_hint = etypes_order_hint
        if import_log is None:
            import_log = SimpleImportLog('<unspecified>')
        self.import_log = import_log
        self.raise_on_error = raise_on_error
        # set of created/updated eids
        self.created = set()
        self.updated = set()

    def import_entities(self, ext_entities):
        """Import given external entities (:class:`ExtEntity`) stream (usually a generator)."""
        # {etype: [etype dict]} of entities that are in the import queue
        queue = {}
        # order entity dictionaries then create/update them
        deferred = self._import_entities(ext_entities, queue)
        # create deferred relations that don't exist already
        missing_relations = self.prepare_insert_deferred_relations(deferred)
        self._warn_about_missing_work(queue, missing_relations)

    def _import_entities(self, ext_entities, queue):
        extid2eid = self.extid2eid
        deferred = {}  # non inlined relations that may be deferred
        self.import_log.record_debug('importing entities')
        for ext_entity in self.iter_ext_entities(ext_entities, deferred, queue):
            try:
                eid = extid2eid[ext_entity.extid]
            except KeyError:
                self.prepare_insert_entity(ext_entity)
            else:
                if ext_entity.values:
                    self.prepare_update_entity(ext_entity, eid)
        return deferred

    def iter_ext_entities(self, ext_entities, deferred, queue):
        """Yield external entities in an order which attempts to satisfy
        schema constraints (inlined / cardinality) and to optimize the import.
        """
        schema = self.schema
        extid2eid = self.extid2eid
        for ext_entity in ext_entities:
            # check data in the transitional representation and prepare it for
            # later insertion in the database
            for subject_uri, rtype, object_uri in ext_entity.prepare(schema):
                deferred.setdefault(rtype, set()).add((subject_uri, object_uri))
            if not ext_entity.is_ready(extid2eid):
                queue.setdefault(ext_entity.etype, []).append(ext_entity)
                continue
            yield ext_entity
            # check for some entities in the queue that may now be ready. We'll have to restart
            # search for ready entities until no one is generated
            new = True
            while new:
                new = False
                for etype in self.etypes_order_hint:
                    if etype in queue:
                        new_queue = []
                        for ext_entity in queue[etype]:
                            if ext_entity.is_ready(extid2eid):
                                yield ext_entity
                                # may unlock entity previously handled within this loop
                                new = True
                            else:
                                new_queue.append(ext_entity)
                        if new_queue:
                            queue[etype][:] = new_queue
                        else:
                            del queue[etype]

    def prepare_insert_entity(self, ext_entity):
        """Call the store to prepare insertion of the given external entity"""
        eid = self.store.prepare_insert_entity(ext_entity.etype, **ext_entity.values)
        self.extid2eid[ext_entity.extid] = eid
        self.created.add(eid)
        return eid

    def prepare_update_entity(self, ext_entity, eid):
        """Call the store to prepare update of the given external entity"""
        self.store.prepare_update_entity(ext_entity.etype, eid, **ext_entity.values)
        self.updated.add(eid)

    def prepare_insert_deferred_relations(self, deferred):
        """Call the store to insert deferred relations (not handled during insertion/update for
        entities). Return a list of relations `[(subj ext id, obj ext id)]` that may not be inserted
        because the target entities don't exists yet.
        """
        prepare_insert_relation = self.store.prepare_insert_relation
        rschema = self.schema.rschema
        extid2eid = self.extid2eid
        missing_relations = []
        for rtype, relations in deferred.items():
            self.import_log.record_debug('importing %s %s relations' % (len(relations), rtype))
            symmetric = rschema(rtype).symmetric
            existing = self.existing_relations[rtype]
            for subject_uri, object_uri in relations:
                try:
                    subject_eid = extid2eid[subject_uri]
                    object_eid = extid2eid[object_uri]
                except KeyError:
                    missing_relations.append((subject_uri, rtype, object_uri))
                    continue
                if (subject_eid, object_eid) not in existing:
                    prepare_insert_relation(subject_eid, rtype, object_eid)
                    existing.add((subject_eid, object_eid))
                    if symmetric:
                        existing.add((object_eid, subject_eid))
        return missing_relations

    def _warn_about_missing_work(self, queue, missing_relations):
        error = self.import_log.record_error
        if queue:
            msgs = ["can't create some entities, is there some cycle or "
                    "missing data?"]
            for ext_entities in queue.values():
                for ext_entity in ext_entities:
                    msgs.append(str(ext_entity))
            map(error, msgs)
            if self.raise_on_error:
                raise Exception('\n'.join(msgs))
        if missing_relations:
            msgs = ["can't create some relations, is there missing data?"]
            for subject_uri, rtype, object_uri in missing_relations:
                msgs.append("%s %s %s" % (subject_uri, rtype, object_uri))
            map(error, msgs)
            if self.raise_on_error:
                raise Exception('\n'.join(msgs))


class SimpleImportLog(object):
    """Fake CWDataImport log using a simple text format.

    Useful to display logs in the UI instead of storing them to the
    database.
    """

    def __init__(self, filename):
        self.logs = []
        self.filename = filename

    def record_debug(self, msg, path=None, line=None):
        self._log(logging.DEBUG, msg, path, line)

    def record_info(self, msg, path=None, line=None):
        self._log(logging.INFO, msg, path, line)

    def record_warning(self, msg, path=None, line=None):
        self._log(logging.WARNING, msg, path, line)

    def record_error(self, msg, path=None, line=None):
        self._log(logging.ERROR, msg, path, line)

    def record_fatal(self, msg, path=None, line=None):
        self._log(logging.FATAL, msg, path, line)

    def _log(self, severity, msg, path, line):
        encodedmsg = u'%s\t%s\t%s\t%s' % (severity, self.filename,
                                          line or u'', msg)
        self.logs.append(encodedmsg)


class HTMLImportLog(SimpleImportLog):
    """Fake CWDataImport log using a simple HTML format."""
    def __init__(self, filename):
        super(HTMLImportLog, self).__init__(xml_escape(filename))

    def _log(self, severity, msg, path, line):
        encodedmsg = u'%s\t%s\t%s\t%s<br/>' % (severity, self.filename,
                                               line or u'', xml_escape(msg))
        self.logs.append(encodedmsg)