# copyright 2003-2015 LOGILAB S.A. (Paris, FRANCE), all rights reserved.# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr## This file is part of CubicWeb.## CubicWeb is free software: you can redistribute it and/or modify it under the# terms of the GNU Lesser General Public License as published by the Free# Software Foundation, either version 2.1 of the License, or (at your option)# any later version.## CubicWeb is distributed in the hope that it will be useful, but WITHOUT# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more# details.## You should have received a copy of the GNU Lesser General Public License along# with CubicWeb. If not, see <http://www.gnu.org/licenses/>."""Functions to help importing CSV data"""from__future__importabsolute_import,print_functionimportcsvascsvmodimportwarningsimportos.pathasospfromsiximportstring_typesfromlogilab.commonimportshellutilsdefcount_lines(stream_or_filename):ifisinstance(stream_or_filename,string_types):f=open(stream_or_filename)else:f=stream_or_filenamef.seek(0)fori,lineinenumerate(f):passf.seek(0)returni+1defucsvreader_pb(stream_or_path,encoding='utf-8',delimiter=',',quotechar='"',skipfirst=False,withpb=True,skip_empty=True,separator=None,quote=None):"""same as :func:`ucsvreader` but a progress bar is displayed as we iter on rows"""ifseparatorisnotNone:delimiter=separatorwarnings.warn("[3.20] 'separator' kwarg is deprecated, use 'delimiter' instead")ifquoteisnotNone:quotechar=quotewarnings.warn("[3.20] 'quote' kwarg is deprecated, use 'quotechar' instead")ifisinstance(stream_or_path,string_types):ifnotosp.exists(stream_or_path):raiseException("file doesn't exists: %s"%stream_or_path)stream=open(stream_or_path)else:stream=stream_or_pathrowcount=count_lines(stream)ifskipfirst:rowcount-=1ifwithpb:pb=shellutils.ProgressBar(rowcount,50)forurowinucsvreader(stream,encoding,delimiter,quotechar,skipfirst=skipfirst,skip_empty=skip_empty):yieldurowifwithpb:pb.update()print(' %s rows imported'%rowcount)defucsvreader(stream,encoding='utf-8',delimiter=',',quotechar='"',skipfirst=False,ignore_errors=False,skip_empty=True,separator=None,quote=None):"""A csv reader that accepts files with any encoding and outputs unicode strings if skip_empty (the default), lines without any values specified (only separators) will be skipped. This is useful for Excel exports which may be full of such lines. """ifseparatorisnotNone:delimiter=separatorwarnings.warn("[3.20] 'separator' kwarg is deprecated, use 'delimiter' instead")ifquoteisnotNone:quotechar=quotewarnings.warn("[3.20] 'quote' kwarg is deprecated, use 'quotechar' instead")it=iter(csvmod.reader(stream,delimiter=delimiter,quotechar=quotechar))ifnotignore_errors:ifskipfirst:next(it)forrowinit:decoded=[item.decode(encoding)foriteminrow]ifnotskip_emptyorany(decoded):yielddecodedelse:ifskipfirst:try:row=next(it)exceptcsvmod.Error:pass# Safe version, that can cope with error in CSV filewhileTrue:try:row=next(it)# End of CSV, breakexceptStopIteration:break# Error in CSV, ignore line and continueexceptcsvmod.Error:continuedecoded=[item.decode(encoding)foriteminrow]ifnotskip_emptyorany(decoded):yielddecoded