dataimport.py
changeset 9821 2077c8da1893
parent 9770 112c884b2d8d
child 9827 c7ce035aede8
equal deleted inserted replaced
9819:95902c0b991b 9821:2077c8da1893
   139         if skipfirst:
   139         if skipfirst:
   140             it.next()
   140             it.next()
   141         for row in it:
   141         for row in it:
   142             decoded = [item.decode(encoding) for item in row]
   142             decoded = [item.decode(encoding) for item in row]
   143             if not skip_empty or any(decoded):
   143             if not skip_empty or any(decoded):
   144                 yield [item.decode(encoding) for item in row]
   144                 yield decoded
   145     else:
   145     else:
   146         # Skip first line
   146         if skipfirst:
   147         try:
   147             try:
   148             row = it.next()
   148                 row = it.next()
   149         except csv.Error:
   149             except csv.Error:
   150             pass
   150                 pass
   151         # Safe version, that can cope with error in CSV file
   151         # Safe version, that can cope with error in CSV file
   152         while True:
   152         while True:
   153             try:
   153             try:
   154                 row = it.next()
   154                 row = it.next()
   155             # End of CSV, break
   155             # End of CSV, break
   470                 if value is None:
   470                 if value is None:
   471                     return
   471                     return
   472                 if isinstance(value, unicode):
   472                 if isinstance(value, unicode):
   473                     value = value.encode(encoding)
   473                     value = value.encode(encoding)
   474             elif isinstance(value, (date, datetime)):
   474             elif isinstance(value, (date, datetime)):
   475                 # Do not use strftime, as it yields issue
       
   476                 # with date < 1900
       
   477                 value = '%04d-%02d-%02d' % (value.year,
   475                 value = '%04d-%02d-%02d' % (value.year,
   478                                             value.month,
   476                                             value.month,
   479                                             value.day)
   477                                             value.day)
       
   478                 if isinstance(value, datetime):
       
   479                     value += ' %02d:%02d:%02d' % (value.hour,
       
   480                                                   value.minutes,
       
   481                                                   value.second)
   480             else:
   482             else:
   481                 return None
   483                 return None
   482             # We push the value to the new formatted row
   484             # We push the value to the new formatted row
   483             # if the value is not None and could be converted to a string.
   485             # if the value is not None and could be converted to a string.
   484             formatted_row.append(value)
   486             formatted_row.append(value)
   618         eid_from, rtype, eid_to = super(RQLObjectStore, self).relate(
   620         eid_from, rtype, eid_to = super(RQLObjectStore, self).relate(
   619             eid_from, rtype, eid_to, **kwargs)
   621             eid_from, rtype, eid_to, **kwargs)
   620         self.rql('SET X %s Y WHERE X eid %%(x)s, Y eid %%(y)s' % rtype,
   622         self.rql('SET X %s Y WHERE X eid %%(x)s, Y eid %%(y)s' % rtype,
   621                  {'x': int(eid_from), 'y': int(eid_to)})
   623                  {'x': int(eid_from), 'y': int(eid_to)})
   622 
   624 
       
   625     @deprecated("[3.19] use session.find(*args, **kwargs).entities() instead")
   623     def find_entities(self, *args, **kwargs):
   626     def find_entities(self, *args, **kwargs):
   624         return self.session.find_entities(*args, **kwargs)
   627         return self.session.find(*args, **kwargs).entities()
   625 
   628 
       
   629     @deprecated("[3.19] use session.find(*args, **kwargs).one() instead")
   626     def find_one_entity(self, *args, **kwargs):
   630     def find_one_entity(self, *args, **kwargs):
   627         return self.session.find_one_entity(*args, **kwargs)
   631         return self.session.find(*args, **kwargs).one()
   628 
   632 
   629 # the import controller ########################################################
   633 # the import controller ########################################################
   630 
   634 
   631 class CWImportController(object):
   635 class CWImportController(object):
   632     """Controller of the data import process.
   636     """Controller of the data import process.
   856         entity = self.session.vreg['etypes'].etype_class(etype)(self.session)
   860         entity = self.session.vreg['etypes'].etype_class(etype)(self.session)
   857         # entity are "surface" copied, avoid shared dict between copies
   861         # entity are "surface" copied, avoid shared dict between copies
   858         del entity.cw_extra_kwargs
   862         del entity.cw_extra_kwargs
   859         entity.cw_edited = EditedEntity(entity)
   863         entity.cw_edited = EditedEntity(entity)
   860         for attr in self.etype_attrs:
   864         for attr in self.etype_attrs:
   861             entity.cw_edited.edited_attribute(attr, self.generate(entity, attr))
   865             genfunc = self.generate(attr)
       
   866             if genfunc:
       
   867                 entity.cw_edited.edited_attribute(attr, genfunc(entity))
   862         rels = {}
   868         rels = {}
   863         for rel in self.etype_rels:
   869         for rel in self.etype_rels:
   864             rels[rel] = self.generate(entity, rel)
   870             genfunc = self.generate(rel)
       
   871             if genfunc:
       
   872                 rels[rel] = genfunc(entity)
   865         return entity, rels
   873         return entity, rels
   866 
   874 
   867     def init_entity(self, entity):
   875     def init_entity(self, entity):
   868         entity.eid = self.source.create_eid(self.session)
   876         entity.eid = self.source.create_eid(self.session)
   869         for attr in self.entity_attrs:
   877         for attr in self.entity_attrs:
   870             entity.cw_edited.edited_attribute(attr, self.generate(entity, attr))
   878             genfunc = self.generate(attr)
   871 
   879             if genfunc:
   872     def generate(self, entity, rtype):
   880                 entity.cw_edited.edited_attribute(attr, genfunc(entity))
   873         return getattr(self, 'gen_%s' % rtype)(entity)
   881 
       
   882     def generate(self, rtype):
       
   883         return getattr(self, 'gen_%s' % rtype, None)
   874 
   884 
   875     def gen_cwuri(self, entity):
   885     def gen_cwuri(self, entity):
   876         return u'%s%s' % (self.baseurl, entity.eid)
   886         return u'%s%s' % (self.baseurl, entity.eid)
   877 
   887 
   878     def gen_creation_date(self, entity):
   888     def gen_creation_date(self, entity):
   879         return self.time
   889         return self.time
       
   890 
   880     def gen_modification_date(self, entity):
   891     def gen_modification_date(self, entity):
   881         return self.time
   892         return self.time
   882 
   893 
   883     def gen_created_by(self, entity):
   894     def gen_created_by(self, entity):
   884         return self.session.user.eid
   895         return self.session.user.eid
       
   896 
   885     def gen_owned_by(self, entity):
   897     def gen_owned_by(self, entity):
   886         return self.session.user.eid
   898         return self.session.user.eid
   887 
   899 
   888 
   900 
   889 ###########################################################################
   901 ###########################################################################