test/unittest_dataimport.py
author Sylvain Thénault <sylvain.thenault@logilab.fr>
Wed, 02 Sep 2015 15:31:18 +0200
changeset 10634 06a43f727601
parent 10294 277074659cad
child 10349 efbbf1e93a04
permissions -rw-r--r--
[web/views] avoid propagation of NoSelectableObject in some case of inlined relations / permissions When selecting an inlined creation form, we should catch the NoSelectable exception that will be raised if the user cannot add entities of the target type (this is not and cannot be verified earlier) or if some other custom selector prevents the form from being selected. Closes #6510921
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
9903
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
     1
# -*- coding: utf-8 -*-
10294
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
     2
9903
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
     3
import datetime as DT
9181
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     4
from StringIO import StringIO
10294
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
     5
9181
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     6
from logilab.common.testlib import TestCase, unittest_main
10294
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
     7
9181
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     8
from cubicweb import dataimport
9909
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
     9
from cubicweb.devtools.testlib import CubicWebTC
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    10
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    11
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    12
class RQLObjectStoreTC(CubicWebTC):
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    13
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    14
    def test_all(self):
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    15
        with self.admin_access.repo_cnx() as cnx:
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    16
            store = dataimport.RQLObjectStore(cnx)
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    17
            group_eid = store.create_entity('CWGroup', name=u'grp').eid
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    18
            user_eid = store.create_entity('CWUser', login=u'lgn', upassword=u'pwd').eid
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    19
            store.relate(user_eid, 'in_group', group_eid)
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    20
            cnx.commit()
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    21
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    22
        with self.admin_access.repo_cnx() as cnx:
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    23
            users = cnx.execute('CWUser X WHERE X login "lgn"')
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    24
            self.assertEqual(1, len(users))
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    25
            self.assertEqual(user_eid, users.one().eid)
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    26
            groups = cnx.execute('CWGroup X WHERE U in_group X, U login "lgn"')
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    27
            self.assertEqual(1, len(users))
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    28
            self.assertEqual(group_eid, groups.one().eid)
9695
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
    29
10294
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
    30
9903
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    31
class CreateCopyFromBufferTC(TestCase):
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    32
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    33
    # test converters
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    34
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    35
    def test_convert_none(self):
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    36
        cnvt = dataimport._copyfrom_buffer_convert_None
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    37
        self.assertEqual('NULL', cnvt(None))
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    38
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    39
    def test_convert_number(self):
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    40
        cnvt = dataimport._copyfrom_buffer_convert_number
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    41
        self.assertEqual('42', cnvt(42))
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    42
        self.assertEqual('42', cnvt(42L))
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    43
        self.assertEqual('42.42', cnvt(42.42))
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    44
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    45
    def test_convert_string(self):
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    46
        cnvt = dataimport._copyfrom_buffer_convert_string
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    47
        # simple
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    48
        self.assertEqual('babar', cnvt('babar'))
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    49
        # unicode
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    50
        self.assertEqual('\xc3\xa9l\xc3\xa9phant', cnvt(u'éléphant'))
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    51
        self.assertEqual('\xe9l\xe9phant', cnvt(u'éléphant', encoding='latin1'))
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    52
        self.assertEqual('babar#', cnvt('babar\t', replace_sep='#'))
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    53
        self.assertRaises(ValueError, cnvt, 'babar\t')
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    54
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    55
    def test_convert_date(self):
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    56
        cnvt = dataimport._copyfrom_buffer_convert_date
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    57
        self.assertEqual('0666-01-13', cnvt(DT.date(666, 1, 13)))
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    58
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    59
    def test_convert_time(self):
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    60
        cnvt = dataimport._copyfrom_buffer_convert_time
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    61
        self.assertEqual('06:06:06.000100', cnvt(DT.time(6, 6, 6, 100)))
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    62
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    63
    def test_convert_datetime(self):
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    64
        cnvt = dataimport._copyfrom_buffer_convert_datetime
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    65
        self.assertEqual('0666-06-13 06:06:06.000000', cnvt(DT.datetime(666, 6, 13, 6, 6, 6)))
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    66
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    67
    # test buffer
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    68
    def test_create_copyfrom_buffer_tuple(self):
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    69
        cnvt = dataimport._create_copyfrom_buffer
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    70
        data = ((42, 42L, 42.42, u'éléphant', DT.date(666, 1, 13), DT.time(6, 6, 6), DT.datetime(666, 6, 13, 6, 6, 6)),
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    71
                (6, 6L, 6.6, u'babar', DT.date(2014, 1, 14), DT.time(4, 2, 1), DT.datetime(2014, 1, 1, 0, 0, 0)))
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    72
        results = dataimport._create_copyfrom_buffer(data)
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    73
        # all columns
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    74
        expected = '''42\t42\t42.42\téléphant\t0666-01-13\t06:06:06.000000\t0666-06-13 06:06:06.000000
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    75
6\t6\t6.6\tbabar\t2014-01-14\t04:02:01.000000\t2014-01-01 00:00:00.000000'''
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    76
        self.assertMultiLineEqual(expected, results.getvalue())
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    77
        # selected columns
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    78
        results = dataimport._create_copyfrom_buffer(data, columns=(1, 3, 6))
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    79
        expected = '''42\téléphant\t0666-06-13 06:06:06.000000
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    80
6\tbabar\t2014-01-01 00:00:00.000000'''
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    81
        self.assertMultiLineEqual(expected, results.getvalue())
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    82
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    83
    def test_create_copyfrom_buffer_dict(self):
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    84
        cnvt = dataimport._create_copyfrom_buffer
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    85
        data = (dict(integer=42, double=42.42, text=u'éléphant', date=DT.datetime(666, 6, 13, 6, 6, 6)),
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    86
                dict(integer=6, double=6.6, text=u'babar', date=DT.datetime(2014, 1, 1, 0, 0, 0)))
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    87
        results = dataimport._create_copyfrom_buffer(data, ('integer', 'text'))
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    88
        expected = '''42\téléphant\n6\tbabar'''
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    89
        self.assertMultiLineEqual(expected, results.getvalue())
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    90
9695
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
    91
9181
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    92
class UcsvreaderTC(TestCase):
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    93
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    94
    def test_empty_lines_skipped(self):
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    95
        stream = StringIO('''a,b,c,d,
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    96
1,2,3,4,
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    97
,,,,
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    98
,,,,
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    99
''')
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   100
        self.assertEqual([[u'a', u'b', u'c', u'd', u''],
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   101
                          [u'1', u'2', u'3', u'4', u''],
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   102
                          ],
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   103
                         list(dataimport.ucsvreader(stream)))
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   104
        stream.seek(0)
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   105
        self.assertEqual([[u'a', u'b', u'c', u'd', u''],
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   106
                          [u'1', u'2', u'3', u'4', u''],
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   107
                          [u'', u'', u'', u'', u''],
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   108
                          [u'', u'', u'', u'', u'']
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   109
                          ],
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   110
                         list(dataimport.ucsvreader(stream, skip_empty=False)))
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   111
9695
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   112
    def test_skip_first(self):
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   113
        stream = StringIO('a,b,c,d,\n'
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   114
                          '1,2,3,4,\n')
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   115
        reader = dataimport.ucsvreader(stream, skipfirst=True,
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   116
                                       ignore_errors=True)
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   117
        self.assertEqual(list(reader),
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   118
                         [[u'1', u'2', u'3', u'4', u'']])
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   119
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   120
        stream.seek(0)
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   121
        reader = dataimport.ucsvreader(stream, skipfirst=True,
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   122
                                       ignore_errors=False)
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   123
        self.assertEqual(list(reader),
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   124
                         [[u'1', u'2', u'3', u'4', u'']])
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   125
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   126
        stream.seek(0)
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   127
        reader = dataimport.ucsvreader(stream, skipfirst=False,
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   128
                                       ignore_errors=True)
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   129
        self.assertEqual(list(reader),
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   130
                         [[u'a', u'b', u'c', u'd', u''],
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   131
                          [u'1', u'2', u'3', u'4', u'']])
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   132
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   133
        stream.seek(0)
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   134
        reader = dataimport.ucsvreader(stream, skipfirst=False,
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   135
                                       ignore_errors=False)
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   136
        self.assertEqual(list(reader),
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   137
                         [[u'a', u'b', u'c', u'd', u''],
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   138
                          [u'1', u'2', u'3', u'4', u'']])
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   139
9181
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   140
10286
0f8c3ac88f1e [dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9909
diff changeset
   141
class MetaGeneratorTC(CubicWebTC):
10294
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
   142
10286
0f8c3ac88f1e [dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9909
diff changeset
   143
    def test_dont_generate_relation_to_internal_manager(self):
0f8c3ac88f1e [dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9909
diff changeset
   144
        with self.admin_access.repo_cnx() as cnx:
0f8c3ac88f1e [dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9909
diff changeset
   145
            metagen = dataimport.MetaGenerator(cnx)
0f8c3ac88f1e [dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9909
diff changeset
   146
            self.assertIn('created_by', metagen.etype_rels)
0f8c3ac88f1e [dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9909
diff changeset
   147
            self.assertIn('owned_by', metagen.etype_rels)
0f8c3ac88f1e [dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9909
diff changeset
   148
        with self.repo.internal_cnx() as cnx:
0f8c3ac88f1e [dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9909
diff changeset
   149
            metagen = dataimport.MetaGenerator(cnx)
0f8c3ac88f1e [dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9909
diff changeset
   150
            self.assertNotIn('created_by', metagen.etype_rels)
0f8c3ac88f1e [dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9909
diff changeset
   151
            self.assertNotIn('owned_by', metagen.etype_rels)
0f8c3ac88f1e [dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9909
diff changeset
   152
10294
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
   153
    def test_dont_generate_specified_values(self):
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
   154
        with self.admin_access.repo_cnx() as cnx:
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
   155
            metagen = dataimport.MetaGenerator(cnx)
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
   156
            # hijack gen_modification_date to ensure we don't go through it
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
   157
            metagen.gen_modification_date = None
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
   158
            md = DT.datetime.now() - DT.timedelta(days=1)
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
   159
            entity, rels = metagen.base_etype_dicts('CWUser')
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
   160
            entity.cw_edited.update(dict(modification_date=md))
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
   161
            with cnx.ensure_cnx_set:
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
   162
                metagen.init_entity(entity)
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
   163
            self.assertEqual(entity.cw_edited['modification_date'], md)
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
   164
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
   165
9181
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   166
if __name__ == '__main__':
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   167
    unittest_main()