author | Julien Cristau <julien.cristau@logilab.fr> |
Mon, 19 Oct 2015 14:29:06 +0200 | |
changeset 10804 | ee113e1e03de |
parent 10669 | 155c29e0ed1c |
child 10807 | bb0c7dbd1fe7 |
permissions | -rw-r--r-- |
10513
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
1 |
# copyright 2003-2015 LOGILAB S.A. (Paris, FRANCE), all rights reserved. |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
2 |
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
3 |
# |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
4 |
# This file is part of CubicWeb. |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
5 |
# |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
6 |
# CubicWeb is free software: you can redistribute it and/or modify it under the |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
7 |
# terms of the GNU Lesser General Public License as published by the Free |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
8 |
# Software Foundation, either version 2.1 of the License, or (at your option) |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
9 |
# any later version. |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
10 |
# |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
11 |
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
12 |
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
13 |
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
14 |
# details. |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
15 |
# |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
16 |
# You should have received a copy of the GNU Lesser General Public License along |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
17 |
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>. |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
18 |
"""Functions to help importing CSV data""" |
10589
7c23b7de2b8d
[py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10513
diff
changeset
|
19 |
from __future__ import absolute_import, print_function |
10513
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
20 |
|
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
21 |
import csv as csvmod |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
22 |
import warnings |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
23 |
import os.path as osp |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
24 |
|
10612
84468b90e9c1
[py3k] basestring → six.string_types
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10589
diff
changeset
|
25 |
from six import string_types |
84468b90e9c1
[py3k] basestring → six.string_types
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10589
diff
changeset
|
26 |
|
10513
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
27 |
from logilab.common import shellutils |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
28 |
|
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
29 |
|
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
30 |
def count_lines(stream_or_filename): |
10612
84468b90e9c1
[py3k] basestring → six.string_types
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10589
diff
changeset
|
31 |
if isinstance(stream_or_filename, string_types): |
10513
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
32 |
f = open(stream_or_filename) |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
33 |
else: |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
34 |
f = stream_or_filename |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
35 |
f.seek(0) |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
36 |
for i, line in enumerate(f): |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
37 |
pass |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
38 |
f.seek(0) |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
39 |
return i+1 |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
40 |
|
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
41 |
|
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
42 |
def ucsvreader_pb(stream_or_path, encoding='utf-8', delimiter=',', quotechar='"', |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
43 |
skipfirst=False, withpb=True, skip_empty=True, separator=None, |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
44 |
quote=None): |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
45 |
"""same as :func:`ucsvreader` but a progress bar is displayed as we iter on rows""" |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
46 |
if separator is not None: |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
47 |
delimiter = separator |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
48 |
warnings.warn("[3.20] 'separator' kwarg is deprecated, use 'delimiter' instead") |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
49 |
if quote is not None: |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
50 |
quotechar = quote |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
51 |
warnings.warn("[3.20] 'quote' kwarg is deprecated, use 'quotechar' instead") |
10612
84468b90e9c1
[py3k] basestring → six.string_types
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10589
diff
changeset
|
52 |
if isinstance(stream_or_path, string_types): |
10513
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
53 |
if not osp.exists(stream_or_path): |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
54 |
raise Exception("file doesn't exists: %s" % stream_or_path) |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
55 |
stream = open(stream_or_path) |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
56 |
else: |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
57 |
stream = stream_or_path |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
58 |
rowcount = count_lines(stream) |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
59 |
if skipfirst: |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
60 |
rowcount -= 1 |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
61 |
if withpb: |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
62 |
pb = shellutils.ProgressBar(rowcount, 50) |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
63 |
for urow in ucsvreader(stream, encoding, delimiter, quotechar, |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
64 |
skipfirst=skipfirst, skip_empty=skip_empty): |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
65 |
yield urow |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
66 |
if withpb: |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
67 |
pb.update() |
10589
7c23b7de2b8d
[py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10513
diff
changeset
|
68 |
print(' %s rows imported' % rowcount) |
10513
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
69 |
|
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
70 |
|
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
71 |
def ucsvreader(stream, encoding='utf-8', delimiter=',', quotechar='"', |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
72 |
skipfirst=False, ignore_errors=False, skip_empty=True, |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
73 |
separator=None, quote=None): |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
74 |
"""A csv reader that accepts files with any encoding and outputs unicode |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
75 |
strings |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
76 |
|
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
77 |
if skip_empty (the default), lines without any values specified (only |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
78 |
separators) will be skipped. This is useful for Excel exports which may be |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
79 |
full of such lines. |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
80 |
""" |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
81 |
if separator is not None: |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
82 |
delimiter = separator |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
83 |
warnings.warn("[3.20] 'separator' kwarg is deprecated, use 'delimiter' instead") |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
84 |
if quote is not None: |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
85 |
quotechar = quote |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
86 |
warnings.warn("[3.20] 'quote' kwarg is deprecated, use 'quotechar' instead") |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
87 |
it = iter(csvmod.reader(stream, delimiter=delimiter, quotechar=quotechar)) |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
88 |
if not ignore_errors: |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
89 |
if skipfirst: |
10669
155c29e0ed1c
[py3k] use next builtin instead of next method
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10612
diff
changeset
|
90 |
next(it) |
10513
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
91 |
for row in it: |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
92 |
decoded = [item.decode(encoding) for item in row] |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
93 |
if not skip_empty or any(decoded): |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
94 |
yield decoded |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
95 |
else: |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
96 |
if skipfirst: |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
97 |
try: |
10669
155c29e0ed1c
[py3k] use next builtin instead of next method
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10612
diff
changeset
|
98 |
row = next(it) |
10513
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
99 |
except csvmod.Error: |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
100 |
pass |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
101 |
# Safe version, that can cope with error in CSV file |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
102 |
while True: |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
103 |
try: |
10669
155c29e0ed1c
[py3k] use next builtin instead of next method
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10612
diff
changeset
|
104 |
row = next(it) |
10513
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
105 |
# End of CSV, break |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
106 |
except StopIteration: |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
107 |
break |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
108 |
# Error in CSV, ignore line and continue |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
109 |
except csvmod.Error: |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
110 |
continue |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
111 |
decoded = [item.decode(encoding) for item in row] |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
112 |
if not skip_empty or any(decoded): |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
113 |
yield decoded |
7bec01a59f92
[dataimport] dispatch and deprecate old code
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
diff
changeset
|
114 |