test/unittest_dataimport.py
author Rémi Cardona <remi.cardona@logilab.fr>
Fri, 07 Nov 2014 15:33:30 +0100
changeset 10349 efbbf1e93a04
parent 10294 277074659cad
permissions -rw-r--r--
[dataimport] Properly escape strings sent to COPY FROM (closes #5278743) See http://www.postgresql.org/docs/9.1/static/sql-copy.html#AEN64296 for escaping codes.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
9903
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
     1
# -*- coding: utf-8 -*-
10294
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
     2
9903
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
     3
import datetime as DT
9181
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     4
from StringIO import StringIO
10294
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
     5
9181
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     6
from logilab.common.testlib import TestCase, unittest_main
10294
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
     7
9181
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
     8
from cubicweb import dataimport
9909
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
     9
from cubicweb.devtools.testlib import CubicWebTC
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    10
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    11
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    12
class RQLObjectStoreTC(CubicWebTC):
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    13
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    14
    def test_all(self):
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    15
        with self.admin_access.repo_cnx() as cnx:
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    16
            store = dataimport.RQLObjectStore(cnx)
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    17
            group_eid = store.create_entity('CWGroup', name=u'grp').eid
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    18
            user_eid = store.create_entity('CWUser', login=u'lgn', upassword=u'pwd').eid
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    19
            store.relate(user_eid, 'in_group', group_eid)
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    20
            cnx.commit()
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    21
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    22
        with self.admin_access.repo_cnx() as cnx:
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    23
            users = cnx.execute('CWUser X WHERE X login "lgn"')
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    24
            self.assertEqual(1, len(users))
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    25
            self.assertEqual(user_eid, users.one().eid)
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    26
            groups = cnx.execute('CWGroup X WHERE U in_group X, U login "lgn"')
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    27
            self.assertEqual(1, len(users))
e2d42ad10f98 [test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents: 9903
diff changeset
    28
            self.assertEqual(group_eid, groups.one().eid)
9695
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
    29
10294
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
    30
9903
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    31
class CreateCopyFromBufferTC(TestCase):
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    32
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    33
    # test converters
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    34
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    35
    def test_convert_none(self):
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    36
        cnvt = dataimport._copyfrom_buffer_convert_None
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    37
        self.assertEqual('NULL', cnvt(None))
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    38
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    39
    def test_convert_number(self):
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    40
        cnvt = dataimport._copyfrom_buffer_convert_number
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    41
        self.assertEqual('42', cnvt(42))
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    42
        self.assertEqual('42', cnvt(42L))
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    43
        self.assertEqual('42.42', cnvt(42.42))
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    44
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    45
    def test_convert_string(self):
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    46
        cnvt = dataimport._copyfrom_buffer_convert_string
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    47
        # simple
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    48
        self.assertEqual('babar', cnvt('babar'))
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    49
        # unicode
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    50
        self.assertEqual('\xc3\xa9l\xc3\xa9phant', cnvt(u'éléphant'))
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    51
        self.assertEqual('\xe9l\xe9phant', cnvt(u'éléphant', encoding='latin1'))
10349
efbbf1e93a04 [dataimport] Properly escape strings sent to COPY FROM (closes #5278743)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 10294
diff changeset
    52
        # escaping
efbbf1e93a04 [dataimport] Properly escape strings sent to COPY FROM (closes #5278743)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 10294
diff changeset
    53
        self.assertEqual('babar\\tceleste\\n', cnvt('babar\tceleste\n'))
efbbf1e93a04 [dataimport] Properly escape strings sent to COPY FROM (closes #5278743)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 10294
diff changeset
    54
        self.assertEqual(r'C:\\new\tC:\\test', cnvt('C:\\new\tC:\\test'))
9903
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    55
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    56
    def test_convert_date(self):
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    57
        cnvt = dataimport._copyfrom_buffer_convert_date
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    58
        self.assertEqual('0666-01-13', cnvt(DT.date(666, 1, 13)))
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    59
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    60
    def test_convert_time(self):
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    61
        cnvt = dataimport._copyfrom_buffer_convert_time
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    62
        self.assertEqual('06:06:06.000100', cnvt(DT.time(6, 6, 6, 100)))
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    63
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    64
    def test_convert_datetime(self):
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    65
        cnvt = dataimport._copyfrom_buffer_convert_datetime
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    66
        self.assertEqual('0666-06-13 06:06:06.000000', cnvt(DT.datetime(666, 6, 13, 6, 6, 6)))
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    67
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    68
    # test buffer
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    69
    def test_create_copyfrom_buffer_tuple(self):
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    70
        cnvt = dataimport._create_copyfrom_buffer
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    71
        data = ((42, 42L, 42.42, u'éléphant', DT.date(666, 1, 13), DT.time(6, 6, 6), DT.datetime(666, 6, 13, 6, 6, 6)),
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    72
                (6, 6L, 6.6, u'babar', DT.date(2014, 1, 14), DT.time(4, 2, 1), DT.datetime(2014, 1, 1, 0, 0, 0)))
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    73
        results = dataimport._create_copyfrom_buffer(data)
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    74
        # all columns
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    75
        expected = '''42\t42\t42.42\téléphant\t0666-01-13\t06:06:06.000000\t0666-06-13 06:06:06.000000
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    76
6\t6\t6.6\tbabar\t2014-01-14\t04:02:01.000000\t2014-01-01 00:00:00.000000'''
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    77
        self.assertMultiLineEqual(expected, results.getvalue())
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    78
        # selected columns
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    79
        results = dataimport._create_copyfrom_buffer(data, columns=(1, 3, 6))
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    80
        expected = '''42\téléphant\t0666-06-13 06:06:06.000000
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    81
6\tbabar\t2014-01-01 00:00:00.000000'''
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    82
        self.assertMultiLineEqual(expected, results.getvalue())
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    83
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    84
    def test_create_copyfrom_buffer_dict(self):
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    85
        cnvt = dataimport._create_copyfrom_buffer
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    86
        data = (dict(integer=42, double=42.42, text=u'éléphant', date=DT.datetime(666, 6, 13, 6, 6, 6)),
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    87
                dict(integer=6, double=6.6, text=u'babar', date=DT.datetime(2014, 1, 1, 0, 0, 0)))
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    88
        results = dataimport._create_copyfrom_buffer(data, ('integer', 'text'))
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    89
        expected = '''42\téléphant\n6\tbabar'''
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    90
        self.assertMultiLineEqual(expected, results.getvalue())
d1fdbdbab194 [dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents: 9695
diff changeset
    91
9695
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
    92
9181
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    93
class UcsvreaderTC(TestCase):
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    94
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    95
    def test_empty_lines_skipped(self):
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    96
        stream = StringIO('''a,b,c,d,
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    97
1,2,3,4,
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    98
,,,,
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
    99
,,,,
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   100
''')
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   101
        self.assertEqual([[u'a', u'b', u'c', u'd', u''],
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   102
                          [u'1', u'2', u'3', u'4', u''],
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   103
                          ],
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   104
                         list(dataimport.ucsvreader(stream)))
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   105
        stream.seek(0)
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   106
        self.assertEqual([[u'a', u'b', u'c', u'd', u''],
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   107
                          [u'1', u'2', u'3', u'4', u''],
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   108
                          [u'', u'', u'', u'', u''],
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   109
                          [u'', u'', u'', u'', u'']
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   110
                          ],
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   111
                         list(dataimport.ucsvreader(stream, skip_empty=False)))
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   112
9695
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   113
    def test_skip_first(self):
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   114
        stream = StringIO('a,b,c,d,\n'
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   115
                          '1,2,3,4,\n')
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   116
        reader = dataimport.ucsvreader(stream, skipfirst=True,
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   117
                                       ignore_errors=True)
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   118
        self.assertEqual(list(reader),
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   119
                         [[u'1', u'2', u'3', u'4', u'']])
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   120
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   121
        stream.seek(0)
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   122
        reader = dataimport.ucsvreader(stream, skipfirst=True,
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   123
                                       ignore_errors=False)
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   124
        self.assertEqual(list(reader),
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   125
                         [[u'1', u'2', u'3', u'4', u'']])
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   126
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   127
        stream.seek(0)
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   128
        reader = dataimport.ucsvreader(stream, skipfirst=False,
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   129
                                       ignore_errors=True)
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   130
        self.assertEqual(list(reader),
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   131
                         [[u'a', u'b', u'c', u'd', u''],
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   132
                          [u'1', u'2', u'3', u'4', u'']])
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   133
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   134
        stream.seek(0)
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   135
        reader = dataimport.ucsvreader(stream, skipfirst=False,
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   136
                                       ignore_errors=False)
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   137
        self.assertEqual(list(reader),
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   138
                         [[u'a', u'b', u'c', u'd', u''],
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   139
                          [u'1', u'2', u'3', u'4', u'']])
aa982b7c3f2a [dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents: 9181
diff changeset
   140
9181
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   141
10286
0f8c3ac88f1e [dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9909
diff changeset
   142
class MetaGeneratorTC(CubicWebTC):
10294
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
   143
10286
0f8c3ac88f1e [dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9909
diff changeset
   144
    def test_dont_generate_relation_to_internal_manager(self):
0f8c3ac88f1e [dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9909
diff changeset
   145
        with self.admin_access.repo_cnx() as cnx:
0f8c3ac88f1e [dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9909
diff changeset
   146
            metagen = dataimport.MetaGenerator(cnx)
0f8c3ac88f1e [dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9909
diff changeset
   147
            self.assertIn('created_by', metagen.etype_rels)
0f8c3ac88f1e [dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9909
diff changeset
   148
            self.assertIn('owned_by', metagen.etype_rels)
0f8c3ac88f1e [dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9909
diff changeset
   149
        with self.repo.internal_cnx() as cnx:
0f8c3ac88f1e [dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9909
diff changeset
   150
            metagen = dataimport.MetaGenerator(cnx)
0f8c3ac88f1e [dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9909
diff changeset
   151
            self.assertNotIn('created_by', metagen.etype_rels)
0f8c3ac88f1e [dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9909
diff changeset
   152
            self.assertNotIn('owned_by', metagen.etype_rels)
0f8c3ac88f1e [dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 9909
diff changeset
   153
10294
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
   154
    def test_dont_generate_specified_values(self):
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
   155
        with self.admin_access.repo_cnx() as cnx:
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
   156
            metagen = dataimport.MetaGenerator(cnx)
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
   157
            # hijack gen_modification_date to ensure we don't go through it
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
   158
            metagen.gen_modification_date = None
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
   159
            md = DT.datetime.now() - DT.timedelta(days=1)
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
   160
            entity, rels = metagen.base_etype_dicts('CWUser')
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
   161
            entity.cw_edited.update(dict(modification_date=md))
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
   162
            with cnx.ensure_cnx_set:
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
   163
                metagen.init_entity(entity)
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
   164
            self.assertEqual(entity.cw_edited['modification_date'], md)
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
   165
277074659cad [dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 10286
diff changeset
   166
9181
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   167
if __name__ == '__main__':
2eac0aa1d3f6 [dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff changeset
   168
    unittest_main()