dataimport/test/test_stores.py
author Julien Cristau <julien.cristau@logilab.fr>
Mon, 09 Nov 2015 15:22:43 +0100
changeset 10869 575982c948a9
parent 10513 7bec01a59f92
child 11033 63d860a14a17
permissions -rw-r--r--
[dataimport] remove drop_index parameter from massive store "drop_index=False" also implied not dropping any constraints, in particular foreign keys, which meant any attempt to import entities would fail, because we only add metadata (the entities table) after the entity's insertion.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
10513
7bec01a59f92 [dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10459
diff changeset
     1
# copyright 2003-2015 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
7bec01a59f92 [dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10459
diff changeset
     2
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
7bec01a59f92 [dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10459
diff changeset
     3
#
7bec01a59f92 [dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10459
diff changeset
     4
# This file is part of CubicWeb.
7bec01a59f92 [dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10459
diff changeset
     5
#
7bec01a59f92 [dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10459
diff changeset
     6
# CubicWeb is free software: you can redistribute it and/or modify it under the
7bec01a59f92 [dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10459
diff changeset
     7
# terms of the GNU Lesser General Public License as published by the Free
7bec01a59f92 [dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10459
diff changeset
     8
# Software Foundation, either version 2.1 of the License, or (at your option)
7bec01a59f92 [dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10459
diff changeset
     9
# any later version.
7bec01a59f92 [dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10459
diff changeset
    10
#
7bec01a59f92 [dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10459
diff changeset
    11
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
7bec01a59f92 [dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10459
diff changeset
    12
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
7bec01a59f92 [dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10459
diff changeset
    13
# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
7bec01a59f92 [dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10459
diff changeset
    14
# details.
7bec01a59f92 [dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10459
diff changeset
    15
#
7bec01a59f92 [dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10459
diff changeset
    16
# You should have received a copy of the GNU Lesser General Public License along
7bec01a59f92 [dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10459
diff changeset
    17
# with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.
7bec01a59f92 [dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10459
diff changeset
    18
"""unittest for cubicweb.dataimport.stores"""
10294
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
    19
9903
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    20
import datetime as DT
10294
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
    21
10513
7bec01a59f92 [dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10459
diff changeset
    22
from cubicweb.dataimport import stores
9909
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    23
from cubicweb.devtools.testlib import CubicWebTC
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    24
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    25
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    26
class RQLObjectStoreTC(CubicWebTC):
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    27
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    28
    def test_all(self):
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    29
        with self.admin_access.repo_cnx() as cnx:
10513
7bec01a59f92 [dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10459
diff changeset
    30
            store = stores.RQLObjectStore(cnx)
10457
1f5026e7d848 [dataimport] Move stores to new API.
Yann Voté <yann.vote@logilab.fr>
parents: 10350
diff changeset
    31
            # Check data insertion
1f5026e7d848 [dataimport] Move stores to new API.
Yann Voté <yann.vote@logilab.fr>
parents: 10350
diff changeset
    32
            group_eid = store.prepare_insert_entity('CWGroup', name=u'grp')
1f5026e7d848 [dataimport] Move stores to new API.
Yann Voté <yann.vote@logilab.fr>
parents: 10350
diff changeset
    33
            user_eid = store.prepare_insert_entity('CWUser', login=u'lgn',
1f5026e7d848 [dataimport] Move stores to new API.
Yann Voté <yann.vote@logilab.fr>
parents: 10350
diff changeset
    34
                                                   upassword=u'pwd')
10459
5ccc3bd8927e [test] Use store.prepare_insert_relation instead of deprecated relate method
Denis Laxalde <denis.laxalde@logilab.fr>
parents: 10457
diff changeset
    35
            store.prepare_insert_relation(user_eid, 'in_group', group_eid)
9909
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    36
            cnx.commit()
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    37
            users = cnx.execute('CWUser X WHERE X login "lgn"')
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    38
            self.assertEqual(1, len(users))
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    39
            self.assertEqual(user_eid, users.one().eid)
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    40
            groups = cnx.execute('CWGroup X WHERE U in_group X, U login "lgn"')
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    41
            self.assertEqual(1, len(users))
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    42
            self.assertEqual(group_eid, groups.one().eid)
10457
1f5026e7d848 [dataimport] Move stores to new API.
Yann Voté <yann.vote@logilab.fr>
parents: 10350
diff changeset
    43
            # Check data update
1f5026e7d848 [dataimport] Move stores to new API.
Yann Voté <yann.vote@logilab.fr>
parents: 10350
diff changeset
    44
            self.set_description('Check data update')
1f5026e7d848 [dataimport] Move stores to new API.
Yann Voté <yann.vote@logilab.fr>
parents: 10350
diff changeset
    45
            store.prepare_update_entity('CWGroup', group_eid, name=u'new_grp')
1f5026e7d848 [dataimport] Move stores to new API.
Yann Voté <yann.vote@logilab.fr>
parents: 10350
diff changeset
    46
            cnx.commit()
1f5026e7d848 [dataimport] Move stores to new API.
Yann Voté <yann.vote@logilab.fr>
parents: 10350
diff changeset
    47
            group = cnx.execute('CWGroup X WHERE X name "grp"')
1f5026e7d848 [dataimport] Move stores to new API.
Yann Voté <yann.vote@logilab.fr>
parents: 10350
diff changeset
    48
            self.assertEqual(len(group), 0)
1f5026e7d848 [dataimport] Move stores to new API.
Yann Voté <yann.vote@logilab.fr>
parents: 10350
diff changeset
    49
            group = cnx.execute('CWGroup X WHERE X name "new_grp"')
1f5026e7d848 [dataimport] Move stores to new API.
Yann Voté <yann.vote@logilab.fr>
parents: 10350
diff changeset
    50
            self.assertEqual, len(group), 1
1f5026e7d848 [dataimport] Move stores to new API.
Yann Voté <yann.vote@logilab.fr>
parents: 10350
diff changeset
    51
            # Check data update with wrong type
1f5026e7d848 [dataimport] Move stores to new API.
Yann Voté <yann.vote@logilab.fr>
parents: 10350
diff changeset
    52
            with self.assertRaises(AssertionError):
1f5026e7d848 [dataimport] Move stores to new API.
Yann Voté <yann.vote@logilab.fr>
parents: 10350
diff changeset
    53
                store.prepare_update_entity('CWUser', group_eid, name=u'new_user')
1f5026e7d848 [dataimport] Move stores to new API.
Yann Voté <yann.vote@logilab.fr>
parents: 10350
diff changeset
    54
            cnx.commit()
1f5026e7d848 [dataimport] Move stores to new API.
Yann Voté <yann.vote@logilab.fr>
parents: 10350
diff changeset
    55
            group = cnx.execute('CWGroup X WHERE X name "new_user"')
1f5026e7d848 [dataimport] Move stores to new API.
Yann Voté <yann.vote@logilab.fr>
parents: 10350
diff changeset
    56
            self.assertEqual(len(group), 0)
1f5026e7d848 [dataimport] Move stores to new API.
Yann Voté <yann.vote@logilab.fr>
parents: 10350
diff changeset
    57
            group = cnx.execute('CWGroup X WHERE X name "new_grp"')
1f5026e7d848 [dataimport] Move stores to new API.
Yann Voté <yann.vote@logilab.fr>
parents: 10350
diff changeset
    58
            self.assertEqual(len(group), 1)
9695
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
    59
10294
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
    60
10286
0f8c3ac88f1e [dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9909
diff changeset
    61
class MetaGeneratorTC(CubicWebTC):
10294
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
    62
10286
0f8c3ac88f1e [dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9909
diff changeset
    63
    def test_dont_generate_relation_to_internal_manager(self):
0f8c3ac88f1e [dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9909
diff changeset
    64
        with self.admin_access.repo_cnx() as cnx:
10513
7bec01a59f92 [dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10459
diff changeset
    65
            metagen = stores.MetaGenerator(cnx)
10286
0f8c3ac88f1e [dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9909
diff changeset
    66
            self.assertIn('created_by', metagen.etype_rels)
0f8c3ac88f1e [dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9909
diff changeset
    67
            self.assertIn('owned_by', metagen.etype_rels)
0f8c3ac88f1e [dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9909
diff changeset
    68
        with self.repo.internal_cnx() as cnx:
10513
7bec01a59f92 [dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10459
diff changeset
    69
            metagen = stores.MetaGenerator(cnx)
10286
0f8c3ac88f1e [dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9909
diff changeset
    70
            self.assertNotIn('created_by', metagen.etype_rels)
0f8c3ac88f1e [dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9909
diff changeset
    71
            self.assertNotIn('owned_by', metagen.etype_rels)
0f8c3ac88f1e [dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9909
diff changeset
    72
10294
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
    73
    def test_dont_generate_specified_values(self):
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
    74
        with self.admin_access.repo_cnx() as cnx:
10513
7bec01a59f92 [dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10459
diff changeset
    75
            metagen = stores.MetaGenerator(cnx)
10294
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
    76
            # hijack gen_modification_date to ensure we don't go through it
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
    77
            metagen.gen_modification_date = None
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
    78
            md = DT.datetime.now() - DT.timedelta(days=1)
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
    79
            entity, rels = metagen.base_etype_dicts('CWUser')
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
    80
            entity.cw_edited.update(dict(modification_date=md))
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
    81
            with cnx.ensure_cnx_set:
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
    82
                metagen.init_entity(entity)
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
    83
            self.assertEqual(entity.cw_edited['modification_date'], md)
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
    84
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
    85
9181
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    86
if __name__ == '__main__':
10513
7bec01a59f92 [dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10459
diff changeset
    87
    from logilab.common.testlib import unittest_main
9181
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    88
    unittest_main()