18 """Functions to help importing CSV data""" |
18 """Functions to help importing CSV data""" |
19 from __future__ import absolute_import, print_function |
19 from __future__ import absolute_import, print_function |
20 |
20 |
21 import codecs |
21 import codecs |
22 import csv as csvmod |
22 import csv as csvmod |
23 import warnings |
|
24 |
23 |
25 from six import PY2, PY3, string_types |
24 from six import PY2, PY3, string_types |
26 |
25 |
27 from logilab.common import shellutils |
26 from logilab.common import shellutils |
28 |
27 |
39 f.seek(0) |
38 f.seek(0) |
40 return i + 1 |
39 return i + 1 |
41 |
40 |
42 |
41 |
43 def ucsvreader_pb(stream_or_path, encoding='utf-8', delimiter=',', quotechar='"', |
42 def ucsvreader_pb(stream_or_path, encoding='utf-8', delimiter=',', quotechar='"', |
44 skipfirst=False, withpb=True, skip_empty=True, separator=None, |
43 skipfirst=False, withpb=True, skip_empty=True): |
45 quote=None): |
|
46 """same as :func:`ucsvreader` but a progress bar is displayed as we iter on rows""" |
44 """same as :func:`ucsvreader` but a progress bar is displayed as we iter on rows""" |
47 if separator is not None: |
|
48 delimiter = separator |
|
49 warnings.warn("[3.20] 'separator' kwarg is deprecated, use 'delimiter' instead") |
|
50 if quote is not None: |
|
51 quotechar = quote |
|
52 warnings.warn("[3.20] 'quote' kwarg is deprecated, use 'quotechar' instead") |
|
53 if isinstance(stream_or_path, string_types): |
45 if isinstance(stream_or_path, string_types): |
54 stream = open(stream_or_path, 'rb') |
46 stream = open(stream_or_path, 'rb') |
55 else: |
47 else: |
56 stream = stream_or_path |
48 stream = stream_or_path |
57 rowcount = count_lines(stream) |
49 rowcount = count_lines(stream) |
66 pb.update() |
58 pb.update() |
67 print(' %s rows imported' % rowcount) |
59 print(' %s rows imported' % rowcount) |
68 |
60 |
69 |
61 |
70 def ucsvreader(stream, encoding='utf-8', delimiter=',', quotechar='"', |
62 def ucsvreader(stream, encoding='utf-8', delimiter=',', quotechar='"', |
71 skipfirst=False, ignore_errors=False, skip_empty=True, |
63 skipfirst=False, ignore_errors=False, skip_empty=True): |
72 separator=None, quote=None): |
|
73 """A csv reader that accepts files with any encoding and outputs unicode |
64 """A csv reader that accepts files with any encoding and outputs unicode |
74 strings |
65 strings |
75 |
66 |
76 if skip_empty (the default), lines without any values specified (only |
67 if skip_empty (the default), lines without any values specified (only |
77 separators) will be skipped. This is useful for Excel exports which may be |
68 separators) will be skipped. This is useful for Excel exports which may be |
78 full of such lines. |
69 full of such lines. |
79 """ |
70 """ |
80 if PY3: |
71 if PY3: |
81 stream = codecs.getreader(encoding)(stream) |
72 stream = codecs.getreader(encoding)(stream) |
82 if separator is not None: |
|
83 delimiter = separator |
|
84 warnings.warn("[3.20] 'separator' kwarg is deprecated, use 'delimiter' instead") |
|
85 if quote is not None: |
|
86 quotechar = quote |
|
87 warnings.warn("[3.20] 'quote' kwarg is deprecated, use 'quotechar' instead") |
|
88 it = iter(csvmod.reader(stream, delimiter=delimiter, quotechar=quotechar)) |
73 it = iter(csvmod.reader(stream, delimiter=delimiter, quotechar=quotechar)) |
89 if not ignore_errors: |
74 if not ignore_errors: |
90 if skipfirst: |
75 if skipfirst: |
91 next(it) |
76 next(it) |
92 for row in it: |
77 for row in it: |