dataimport/test/unittest_importer.py
author Julien Cristau <julien.cristau@logilab.fr>
Fri, 16 Oct 2015 17:06:28 +0200
changeset 10807 bb0c7dbd1fe7
parent 10514 b29d9904482e
child 10809 359cbdf3a515
permissions -rw-r--r--
[dataimport] fix ucsvreader for python3 The csv module in python3 wants str, so use a StreamReader to feed csv.reader and don't decode the output
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
10460
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
     1
# -*- coding: utf-8 -*-
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
     2
# copyright 2015 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
     3
# contact http://www.logilab.fr -- mailto:contact@logilab.fr
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
     4
#
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
     5
# This program is free software: you can redistribute it and/or modify it under
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
     6
# the terms of the GNU Lesser General Public License as published by the Free
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
     7
# Software Foundation, either version 2.1 of the License, or (at your option)
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
     8
# any later version.
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
     9
#
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    10
# This program is distributed in the hope that it will be useful, but WITHOUT
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    11
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    12
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    13
# details.
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    14
#
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    15
# You should have received a copy of the GNU Lesser General Public License along
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    16
# with this program. If not, see <http://www.gnu.org/licenses/>.
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    17
"""Tests for cubicweb.dataimport.importer"""
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    18
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    19
from collections import defaultdict
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    20
10514
b29d9904482e add use_extid_as_cwuri ext entity transform, that will be often necessary and not so easy to write at once
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10460
diff changeset
    21
from logilab.common.testlib import TestCase, unittest_main
10460
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    22
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    23
from cubicweb import ValidationError
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    24
from cubicweb.devtools.testlib import CubicWebTC
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    25
from cubicweb.dataimport import RQLObjectStore, ucsvreader
10514
b29d9904482e add use_extid_as_cwuri ext entity transform, that will be often necessary and not so easy to write at once
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10460
diff changeset
    26
from cubicweb.dataimport.importer import (ExtEntity, ExtEntitiesImporter, SimpleImportLog,
b29d9904482e add use_extid_as_cwuri ext entity transform, that will be often necessary and not so easy to write at once
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10460
diff changeset
    27
                                          RelationMapping, use_extid_as_cwuri)
10460
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    28
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    29
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    30
class RelationMappingTC(CubicWebTC):
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    31
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    32
    def test_nosource(self):
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    33
        with self.admin_access.repo_cnx() as cnx:
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    34
            alice_eid = cnx.create_entity('Personne', nom=u'alice').eid
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    35
            bob_eid = cnx.create_entity('Personne', nom=u'bob', connait=alice_eid).eid
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    36
            cnx.commit()
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    37
            mapping = RelationMapping(cnx)
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    38
            self.assertEqual(mapping['connait'],
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    39
                             set([(bob_eid, alice_eid), (alice_eid, bob_eid)]))
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    40
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    41
    def test_with_source(self):
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    42
        with self.admin_access.repo_cnx() as cnx:
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    43
            alice_eid = cnx.create_entity('Personne', nom=u'alice').eid
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    44
            bob_eid = cnx.create_entity('Personne', nom=u'bob', connait=alice_eid).eid
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    45
            cnx.commit()
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    46
            mapping = RelationMapping(cnx, cnx.find('CWSource', name=u'system').one())
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    47
            self.assertEqual(mapping['connait'],
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    48
                             set([(bob_eid, alice_eid), (alice_eid, bob_eid)]))
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    49
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    50
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    51
class ExtEntitiesImporterTC(CubicWebTC):
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    52
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    53
    def importer(self, cnx):
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    54
        store = RQLObjectStore(cnx)
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    55
        return ExtEntitiesImporter(self.schema, store, raise_on_error=True)
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    56
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    57
    def test_simple_import(self):
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    58
        with self.admin_access.repo_cnx() as cnx:
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    59
            importer = self.importer(cnx)
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    60
            personne = ExtEntity('Personne', 1, {'nom': set([u'de la lune']),
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    61
                                                 'prenom': set([u'Jean'])})
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    62
            importer.import_entities([personne])
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    63
            cnx.commit()
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    64
            rset = cnx.execute('Any X WHERE X is Personne')
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    65
            entity = rset.get_entity(0, 0)
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    66
            self.assertEqual(entity.nom, u'de la lune')
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    67
            self.assertEqual(entity.prenom, u'Jean')
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    68
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    69
    def test_import_missing_required_attribute(self):
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    70
        """Check import of ext entity with missing required attribute"""
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    71
        with self.admin_access.repo_cnx() as cnx:
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    72
            importer = self.importer(cnx)
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    73
            tag = ExtEntity('Personne', 2, {'prenom': set([u'Jean'])})
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    74
            self.assertRaises(ValidationError, importer.import_entities, [tag])
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    75
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    76
    def test_import_inlined_relation(self):
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    77
        """Check import of ext entities with inlined relation"""
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    78
        with self.admin_access.repo_cnx() as cnx:
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    79
            importer = self.importer(cnx)
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    80
            richelieu = ExtEntity('Personne', 3, {'nom': set([u'Richelieu']),
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    81
                                                  'enfant': set([4])})
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    82
            athos = ExtEntity('Personne', 4, {'nom': set([u'Athos'])})
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    83
            importer.import_entities([athos, richelieu])
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    84
            cnx.commit()
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    85
            rset = cnx.execute('Any X WHERE X is Personne, X nom "Richelieu"')
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    86
            entity = rset.get_entity(0, 0)
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    87
            self.assertEqual(entity.enfant[0].nom, 'Athos')
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    88
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    89
    def test_import_non_inlined_relation(self):
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    90
        """Check import of ext entities with non inlined relation"""
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    91
        with self.admin_access.repo_cnx() as cnx:
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    92
            importer = self.importer(cnx)
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    93
            richelieu = ExtEntity('Personne', 5, {'nom': set([u'Richelieu']),
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    94
                                                  'connait': set([6])})
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    95
            athos = ExtEntity('Personne', 6, {'nom': set([u'Athos'])})
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    96
            importer.import_entities([athos, richelieu])
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    97
            cnx.commit()
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    98
            rset = cnx.execute('Any X WHERE X is Personne, X nom "Richelieu"')
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
    99
            entity = rset.get_entity(0, 0)
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   100
            self.assertEqual(entity.connait[0].nom, 'Athos')
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   101
            rset = cnx.execute('Any X WHERE X is Personne, X nom "Athos"')
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   102
            entity = rset.get_entity(0, 0)
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   103
            self.assertEqual(entity.connait[0].nom, 'Richelieu')
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   104
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   105
    def test_import_missing_inlined_relation(self):
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   106
        """Check import of ext entity with missing inlined relation"""
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   107
        with self.admin_access.repo_cnx() as cnx:
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   108
            importer = self.importer(cnx)
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   109
            richelieu = ExtEntity('Personne', 7,
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   110
                                  {'nom': set([u'Richelieu']), 'enfant': set([8])})
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   111
            self.assertRaises(Exception, importer.import_entities, [richelieu])
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   112
            cnx.commit()
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   113
            rset = cnx.execute('Any X WHERE X is Personne, X nom "Richelieu"')
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   114
            self.assertEqual(len(rset), 0)
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   115
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   116
    def test_import_missing_non_inlined_relation(self):
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   117
        """Check import of ext entity with missing non-inlined relation"""
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   118
        with self.admin_access.repo_cnx() as cnx:
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   119
            importer = self.importer(cnx)
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   120
            richelieu = ExtEntity('Personne', 9,
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   121
                                  {'nom': set([u'Richelieu']), 'connait': set([10])})
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   122
            self.assertRaises(Exception, importer.import_entities, [richelieu])
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   123
            cnx.commit()
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   124
            rset = cnx.execute('Any X WHERE X is Personne, X nom "Richelieu"')
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   125
            entity = rset.get_entity(0, 0)
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   126
            self.assertEqual(entity.nom, u'Richelieu')
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   127
            self.assertEqual(len(entity.connait), 0)
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   128
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   129
    def test_update(self):
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   130
        """Check update of ext entity"""
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   131
        with self.admin_access.repo_cnx() as cnx:
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   132
            importer = self.importer(cnx)
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   133
            # First import
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   134
            richelieu = ExtEntity('Personne', 11,
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   135
                                  {'nom': {u'Richelieu Diacre'}})
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   136
            importer.import_entities([richelieu])
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   137
            cnx.commit()
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   138
            rset = cnx.execute('Any X WHERE X is Personne')
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   139
            entity = rset.get_entity(0, 0)
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   140
            self.assertEqual(entity.nom, u'Richelieu Diacre')
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   141
            # Second import
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   142
            richelieu = ExtEntity('Personne', 11,
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   143
                                  {'nom': {u'Richelieu Cardinal'}})
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   144
            importer.import_entities([richelieu])
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   145
            cnx.commit()
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   146
            rset = cnx.execute('Any X WHERE X is Personne')
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   147
            self.assertEqual(len(rset), 1)
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   148
            entity = rset.get_entity(0, 0)
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   149
            self.assertEqual(entity.nom, u'Richelieu Cardinal')
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   150
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   151
10514
b29d9904482e add use_extid_as_cwuri ext entity transform, that will be often necessary and not so easy to write at once
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10460
diff changeset
   152
class UseExtidAsCwuriTC(TestCase):
b29d9904482e add use_extid_as_cwuri ext entity transform, that will be often necessary and not so easy to write at once
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10460
diff changeset
   153
b29d9904482e add use_extid_as_cwuri ext entity transform, that will be often necessary and not so easy to write at once
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10460
diff changeset
   154
    def test(self):
b29d9904482e add use_extid_as_cwuri ext entity transform, that will be often necessary and not so easy to write at once
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10460
diff changeset
   155
        personne = ExtEntity('Personne', 1, {'nom': set([u'de la lune']),
b29d9904482e add use_extid_as_cwuri ext entity transform, that will be often necessary and not so easy to write at once
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10460
diff changeset
   156
                                             'prenom': set([u'Jean'])})
b29d9904482e add use_extid_as_cwuri ext entity transform, that will be often necessary and not so easy to write at once
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10460
diff changeset
   157
        mapping = {}
b29d9904482e add use_extid_as_cwuri ext entity transform, that will be often necessary and not so easy to write at once
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10460
diff changeset
   158
        set_cwuri = use_extid_as_cwuri(mapping)
b29d9904482e add use_extid_as_cwuri ext entity transform, that will be often necessary and not so easy to write at once
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10460
diff changeset
   159
        list(set_cwuri((personne,)))
b29d9904482e add use_extid_as_cwuri ext entity transform, that will be often necessary and not so easy to write at once
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10460
diff changeset
   160
        self.assertIn('cwuri', personne.values)
b29d9904482e add use_extid_as_cwuri ext entity transform, that will be often necessary and not so easy to write at once
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10460
diff changeset
   161
        self.assertEqual(personne.values['cwuri'], set(['1']))
b29d9904482e add use_extid_as_cwuri ext entity transform, that will be often necessary and not so easy to write at once
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10460
diff changeset
   162
        mapping[1] = 'whatever'
b29d9904482e add use_extid_as_cwuri ext entity transform, that will be often necessary and not so easy to write at once
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10460
diff changeset
   163
        personne.values.pop('cwuri')
b29d9904482e add use_extid_as_cwuri ext entity transform, that will be often necessary and not so easy to write at once
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10460
diff changeset
   164
        list(set_cwuri((personne,)))
b29d9904482e add use_extid_as_cwuri ext entity transform, that will be often necessary and not so easy to write at once
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10460
diff changeset
   165
        self.assertNotIn('cwuri', personne.values)
b29d9904482e add use_extid_as_cwuri ext entity transform, that will be often necessary and not so easy to write at once
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10460
diff changeset
   166
b29d9904482e add use_extid_as_cwuri ext entity transform, that will be often necessary and not so easy to write at once
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10460
diff changeset
   167
10460
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   168
def extentities_from_csv(fpath):
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   169
    """Yield ExtEntity read from `fpath` CSV file."""
10807
bb0c7dbd1fe7 [dataimport] fix ucsvreader for python3
Julien Cristau <julien.cristau@logilab.fr>
parents: 10514
diff changeset
   170
    with open(fpath, 'rb') as f:
10460
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   171
        for uri, name, knows in ucsvreader(f, skipfirst=True, skip_empty=False):
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   172
            yield ExtEntity('Personne', uri,
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   173
                            {'nom': set([name]), 'connait': set([knows])})
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   174
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   175
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   176
class DataimportFunctionalTC(CubicWebTC):
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   177
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   178
    def test_csv(self):
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   179
        extenties = extentities_from_csv(self.datapath('people.csv'))
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   180
        with self.admin_access.repo_cnx() as cnx:
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   181
            store = RQLObjectStore(cnx)
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   182
            importer = ExtEntitiesImporter(self.schema, store)
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   183
            importer.import_entities(extenties)
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   184
            cnx.commit()
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   185
            rset = cnx.execute('String N WHERE X nom N, X connait Y, Y nom "Alice"')
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   186
            self.assertEqual(rset[0][0], u'Bob')
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   187
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   188
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   189
if __name__ == '__main__':
d260722f2453 [dataimport] introduce the importer and extentity classes
Yann Voté <yann.vote@logilab.fr>
parents:
diff changeset
   190
    unittest_main()