author | David Douard <david.douard@logilab.fr> |
Mon, 09 May 2016 17:05:14 +0200 | |
changeset 11235 | 636a83e65870 |
parent 10294 | 277074659cad |
child 10349 | efbbf1e93a04 |
permissions | -rw-r--r-- |
9903
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
1 |
# -*- coding: utf-8 -*- |
10294
277074659cad
[dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10286
diff
changeset
|
2 |
|
9903
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
3 |
import datetime as DT |
9181
2eac0aa1d3f6
[dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
4 |
from StringIO import StringIO |
10294
277074659cad
[dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10286
diff
changeset
|
5 |
|
9181
2eac0aa1d3f6
[dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
6 |
from logilab.common.testlib import TestCase, unittest_main |
10294
277074659cad
[dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10286
diff
changeset
|
7 |
|
9181
2eac0aa1d3f6
[dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
8 |
from cubicweb import dataimport |
9909
e2d42ad10f98
[test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents:
9903
diff
changeset
|
9 |
from cubicweb.devtools.testlib import CubicWebTC |
e2d42ad10f98
[test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents:
9903
diff
changeset
|
10 |
|
e2d42ad10f98
[test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents:
9903
diff
changeset
|
11 |
|
e2d42ad10f98
[test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents:
9903
diff
changeset
|
12 |
class RQLObjectStoreTC(CubicWebTC): |
e2d42ad10f98
[test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents:
9903
diff
changeset
|
13 |
|
e2d42ad10f98
[test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents:
9903
diff
changeset
|
14 |
def test_all(self): |
e2d42ad10f98
[test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents:
9903
diff
changeset
|
15 |
with self.admin_access.repo_cnx() as cnx: |
e2d42ad10f98
[test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents:
9903
diff
changeset
|
16 |
store = dataimport.RQLObjectStore(cnx) |
e2d42ad10f98
[test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents:
9903
diff
changeset
|
17 |
group_eid = store.create_entity('CWGroup', name=u'grp').eid |
e2d42ad10f98
[test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents:
9903
diff
changeset
|
18 |
user_eid = store.create_entity('CWUser', login=u'lgn', upassword=u'pwd').eid |
e2d42ad10f98
[test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents:
9903
diff
changeset
|
19 |
store.relate(user_eid, 'in_group', group_eid) |
e2d42ad10f98
[test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents:
9903
diff
changeset
|
20 |
cnx.commit() |
e2d42ad10f98
[test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents:
9903
diff
changeset
|
21 |
|
e2d42ad10f98
[test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents:
9903
diff
changeset
|
22 |
with self.admin_access.repo_cnx() as cnx: |
e2d42ad10f98
[test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents:
9903
diff
changeset
|
23 |
users = cnx.execute('CWUser X WHERE X login "lgn"') |
e2d42ad10f98
[test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents:
9903
diff
changeset
|
24 |
self.assertEqual(1, len(users)) |
e2d42ad10f98
[test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents:
9903
diff
changeset
|
25 |
self.assertEqual(user_eid, users.one().eid) |
e2d42ad10f98
[test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents:
9903
diff
changeset
|
26 |
groups = cnx.execute('CWGroup X WHERE U in_group X, U login "lgn"') |
e2d42ad10f98
[test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents:
9903
diff
changeset
|
27 |
self.assertEqual(1, len(users)) |
e2d42ad10f98
[test] Add test for dataimport's RQLObjectStore
Julien Cristau <julien.cristau@logilab.fr>
parents:
9903
diff
changeset
|
28 |
self.assertEqual(group_eid, groups.one().eid) |
9695
aa982b7c3f2a
[dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9181
diff
changeset
|
29 |
|
10294
277074659cad
[dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10286
diff
changeset
|
30 |
|
9903
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
31 |
class CreateCopyFromBufferTC(TestCase): |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
32 |
|
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
33 |
# test converters |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
34 |
|
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
35 |
def test_convert_none(self): |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
36 |
cnvt = dataimport._copyfrom_buffer_convert_None |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
37 |
self.assertEqual('NULL', cnvt(None)) |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
38 |
|
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
39 |
def test_convert_number(self): |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
40 |
cnvt = dataimport._copyfrom_buffer_convert_number |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
41 |
self.assertEqual('42', cnvt(42)) |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
42 |
self.assertEqual('42', cnvt(42L)) |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
43 |
self.assertEqual('42.42', cnvt(42.42)) |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
44 |
|
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
45 |
def test_convert_string(self): |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
46 |
cnvt = dataimport._copyfrom_buffer_convert_string |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
47 |
# simple |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
48 |
self.assertEqual('babar', cnvt('babar')) |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
49 |
# unicode |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
50 |
self.assertEqual('\xc3\xa9l\xc3\xa9phant', cnvt(u'éléphant')) |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
51 |
self.assertEqual('\xe9l\xe9phant', cnvt(u'éléphant', encoding='latin1')) |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
52 |
self.assertEqual('babar#', cnvt('babar\t', replace_sep='#')) |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
53 |
self.assertRaises(ValueError, cnvt, 'babar\t') |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
54 |
|
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
55 |
def test_convert_date(self): |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
56 |
cnvt = dataimport._copyfrom_buffer_convert_date |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
57 |
self.assertEqual('0666-01-13', cnvt(DT.date(666, 1, 13))) |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
58 |
|
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
59 |
def test_convert_time(self): |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
60 |
cnvt = dataimport._copyfrom_buffer_convert_time |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
61 |
self.assertEqual('06:06:06.000100', cnvt(DT.time(6, 6, 6, 100))) |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
62 |
|
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
63 |
def test_convert_datetime(self): |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
64 |
cnvt = dataimport._copyfrom_buffer_convert_datetime |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
65 |
self.assertEqual('0666-06-13 06:06:06.000000', cnvt(DT.datetime(666, 6, 13, 6, 6, 6))) |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
66 |
|
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
67 |
# test buffer |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
68 |
def test_create_copyfrom_buffer_tuple(self): |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
69 |
cnvt = dataimport._create_copyfrom_buffer |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
70 |
data = ((42, 42L, 42.42, u'éléphant', DT.date(666, 1, 13), DT.time(6, 6, 6), DT.datetime(666, 6, 13, 6, 6, 6)), |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
71 |
(6, 6L, 6.6, u'babar', DT.date(2014, 1, 14), DT.time(4, 2, 1), DT.datetime(2014, 1, 1, 0, 0, 0))) |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
72 |
results = dataimport._create_copyfrom_buffer(data) |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
73 |
# all columns |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
74 |
expected = '''42\t42\t42.42\téléphant\t0666-01-13\t06:06:06.000000\t0666-06-13 06:06:06.000000 |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
75 |
6\t6\t6.6\tbabar\t2014-01-14\t04:02:01.000000\t2014-01-01 00:00:00.000000''' |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
76 |
self.assertMultiLineEqual(expected, results.getvalue()) |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
77 |
# selected columns |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
78 |
results = dataimport._create_copyfrom_buffer(data, columns=(1, 3, 6)) |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
79 |
expected = '''42\téléphant\t0666-06-13 06:06:06.000000 |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
80 |
6\tbabar\t2014-01-01 00:00:00.000000''' |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
81 |
self.assertMultiLineEqual(expected, results.getvalue()) |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
82 |
|
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
83 |
def test_create_copyfrom_buffer_dict(self): |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
84 |
cnvt = dataimport._create_copyfrom_buffer |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
85 |
data = (dict(integer=42, double=42.42, text=u'éléphant', date=DT.datetime(666, 6, 13, 6, 6, 6)), |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
86 |
dict(integer=6, double=6.6, text=u'babar', date=DT.datetime(2014, 1, 1, 0, 0, 0))) |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
87 |
results = dataimport._create_copyfrom_buffer(data, ('integer', 'text')) |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
88 |
expected = '''42\téléphant\n6\tbabar''' |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
89 |
self.assertMultiLineEqual(expected, results.getvalue()) |
d1fdbdbab194
[dataimport] _create_copyfrom_buffer: add the tests
Alain Leufroy <alain.leufroy@logilab.fr>
parents:
9695
diff
changeset
|
90 |
|
9695
aa982b7c3f2a
[dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9181
diff
changeset
|
91 |
|
9181
2eac0aa1d3f6
[dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
92 |
class UcsvreaderTC(TestCase): |
2eac0aa1d3f6
[dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
93 |
|
2eac0aa1d3f6
[dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
94 |
def test_empty_lines_skipped(self): |
2eac0aa1d3f6
[dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
95 |
stream = StringIO('''a,b,c,d, |
2eac0aa1d3f6
[dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
96 |
1,2,3,4, |
2eac0aa1d3f6
[dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
97 |
,,,, |
2eac0aa1d3f6
[dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
98 |
,,,, |
2eac0aa1d3f6
[dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
99 |
''') |
2eac0aa1d3f6
[dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
100 |
self.assertEqual([[u'a', u'b', u'c', u'd', u''], |
2eac0aa1d3f6
[dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
101 |
[u'1', u'2', u'3', u'4', u''], |
2eac0aa1d3f6
[dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
102 |
], |
2eac0aa1d3f6
[dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
103 |
list(dataimport.ucsvreader(stream))) |
2eac0aa1d3f6
[dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
104 |
stream.seek(0) |
2eac0aa1d3f6
[dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
105 |
self.assertEqual([[u'a', u'b', u'c', u'd', u''], |
2eac0aa1d3f6
[dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
106 |
[u'1', u'2', u'3', u'4', u''], |
2eac0aa1d3f6
[dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
107 |
[u'', u'', u'', u'', u''], |
2eac0aa1d3f6
[dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
108 |
[u'', u'', u'', u'', u''] |
2eac0aa1d3f6
[dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
109 |
], |
2eac0aa1d3f6
[dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
110 |
list(dataimport.ucsvreader(stream, skip_empty=False))) |
2eac0aa1d3f6
[dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
111 |
|
9695
aa982b7c3f2a
[dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9181
diff
changeset
|
112 |
def test_skip_first(self): |
aa982b7c3f2a
[dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9181
diff
changeset
|
113 |
stream = StringIO('a,b,c,d,\n' |
aa982b7c3f2a
[dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9181
diff
changeset
|
114 |
'1,2,3,4,\n') |
aa982b7c3f2a
[dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9181
diff
changeset
|
115 |
reader = dataimport.ucsvreader(stream, skipfirst=True, |
aa982b7c3f2a
[dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9181
diff
changeset
|
116 |
ignore_errors=True) |
aa982b7c3f2a
[dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9181
diff
changeset
|
117 |
self.assertEqual(list(reader), |
aa982b7c3f2a
[dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9181
diff
changeset
|
118 |
[[u'1', u'2', u'3', u'4', u'']]) |
aa982b7c3f2a
[dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9181
diff
changeset
|
119 |
|
aa982b7c3f2a
[dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9181
diff
changeset
|
120 |
stream.seek(0) |
aa982b7c3f2a
[dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9181
diff
changeset
|
121 |
reader = dataimport.ucsvreader(stream, skipfirst=True, |
aa982b7c3f2a
[dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9181
diff
changeset
|
122 |
ignore_errors=False) |
aa982b7c3f2a
[dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9181
diff
changeset
|
123 |
self.assertEqual(list(reader), |
aa982b7c3f2a
[dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9181
diff
changeset
|
124 |
[[u'1', u'2', u'3', u'4', u'']]) |
aa982b7c3f2a
[dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9181
diff
changeset
|
125 |
|
aa982b7c3f2a
[dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9181
diff
changeset
|
126 |
stream.seek(0) |
aa982b7c3f2a
[dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9181
diff
changeset
|
127 |
reader = dataimport.ucsvreader(stream, skipfirst=False, |
aa982b7c3f2a
[dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9181
diff
changeset
|
128 |
ignore_errors=True) |
aa982b7c3f2a
[dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9181
diff
changeset
|
129 |
self.assertEqual(list(reader), |
aa982b7c3f2a
[dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9181
diff
changeset
|
130 |
[[u'a', u'b', u'c', u'd', u''], |
aa982b7c3f2a
[dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9181
diff
changeset
|
131 |
[u'1', u'2', u'3', u'4', u'']]) |
aa982b7c3f2a
[dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9181
diff
changeset
|
132 |
|
aa982b7c3f2a
[dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9181
diff
changeset
|
133 |
stream.seek(0) |
aa982b7c3f2a
[dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9181
diff
changeset
|
134 |
reader = dataimport.ucsvreader(stream, skipfirst=False, |
aa982b7c3f2a
[dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9181
diff
changeset
|
135 |
ignore_errors=False) |
aa982b7c3f2a
[dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9181
diff
changeset
|
136 |
self.assertEqual(list(reader), |
aa982b7c3f2a
[dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9181
diff
changeset
|
137 |
[[u'a', u'b', u'c', u'd', u''], |
aa982b7c3f2a
[dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9181
diff
changeset
|
138 |
[u'1', u'2', u'3', u'4', u'']]) |
aa982b7c3f2a
[dataimport] Prevent ucsvreader from skipping the first line when ignore_errors is True (closes #3705791)
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9181
diff
changeset
|
139 |
|
9181
2eac0aa1d3f6
[dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
140 |
|
10286
0f8c3ac88f1e
[dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9909
diff
changeset
|
141 |
class MetaGeneratorTC(CubicWebTC): |
10294
277074659cad
[dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10286
diff
changeset
|
142 |
|
10286
0f8c3ac88f1e
[dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9909
diff
changeset
|
143 |
def test_dont_generate_relation_to_internal_manager(self): |
0f8c3ac88f1e
[dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9909
diff
changeset
|
144 |
with self.admin_access.repo_cnx() as cnx: |
0f8c3ac88f1e
[dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9909
diff
changeset
|
145 |
metagen = dataimport.MetaGenerator(cnx) |
0f8c3ac88f1e
[dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9909
diff
changeset
|
146 |
self.assertIn('created_by', metagen.etype_rels) |
0f8c3ac88f1e
[dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9909
diff
changeset
|
147 |
self.assertIn('owned_by', metagen.etype_rels) |
0f8c3ac88f1e
[dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9909
diff
changeset
|
148 |
with self.repo.internal_cnx() as cnx: |
0f8c3ac88f1e
[dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9909
diff
changeset
|
149 |
metagen = dataimport.MetaGenerator(cnx) |
0f8c3ac88f1e
[dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9909
diff
changeset
|
150 |
self.assertNotIn('created_by', metagen.etype_rels) |
0f8c3ac88f1e
[dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9909
diff
changeset
|
151 |
self.assertNotIn('owned_by', metagen.etype_rels) |
0f8c3ac88f1e
[dataimport] don't insert created_by / owned_by relations when user is the internal manager (eid = -1).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9909
diff
changeset
|
152 |
|
10294
277074659cad
[dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10286
diff
changeset
|
153 |
def test_dont_generate_specified_values(self): |
277074659cad
[dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10286
diff
changeset
|
154 |
with self.admin_access.repo_cnx() as cnx: |
277074659cad
[dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10286
diff
changeset
|
155 |
metagen = dataimport.MetaGenerator(cnx) |
277074659cad
[dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10286
diff
changeset
|
156 |
# hijack gen_modification_date to ensure we don't go through it |
277074659cad
[dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10286
diff
changeset
|
157 |
metagen.gen_modification_date = None |
277074659cad
[dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10286
diff
changeset
|
158 |
md = DT.datetime.now() - DT.timedelta(days=1) |
277074659cad
[dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10286
diff
changeset
|
159 |
entity, rels = metagen.base_etype_dicts('CWUser') |
277074659cad
[dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10286
diff
changeset
|
160 |
entity.cw_edited.update(dict(modification_date=md)) |
277074659cad
[dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10286
diff
changeset
|
161 |
with cnx.ensure_cnx_set: |
277074659cad
[dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10286
diff
changeset
|
162 |
metagen.init_entity(entity) |
277074659cad
[dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10286
diff
changeset
|
163 |
self.assertEqual(entity.cw_edited['modification_date'], md) |
277074659cad
[dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10286
diff
changeset
|
164 |
|
277074659cad
[dataimport] don't generate metadata which are explicitly specified
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10286
diff
changeset
|
165 |
|
9181
2eac0aa1d3f6
[dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
166 |
if __name__ == '__main__': |
2eac0aa1d3f6
[dataimport] ucsvreader should skip empty lines unless specified otherwise. Closes #3035944
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
167 |
unittest_main() |