# HG changeset patch # User Sylvain Thénault # Date 1454001524 -3600 # Node ID 682b15eb2dd277d167c9c7fe31c1940bea0c2757 # Parent 3a83759854eed2543bf6875415ab0a9f5aeb8448 [dataimport] flake8 diff -r 3a83759854ee -r 682b15eb2dd2 cubicweb/dataimport/csv.py --- a/cubicweb/dataimport/csv.py Thu Jan 28 14:02:31 2016 +0100 +++ b/cubicweb/dataimport/csv.py Thu Jan 28 18:18:44 2016 +0100 @@ -21,7 +21,6 @@ import codecs import csv as csvmod import warnings -import os.path as osp from six import PY2, PY3, string_types @@ -37,7 +36,7 @@ for i, line in enumerate(f): pass f.seek(0) - return i+1 + return i + 1 def ucsvreader_pb(stream_or_path, encoding='utf-8', delimiter=',', quotechar='"', diff -r 3a83759854ee -r 682b15eb2dd2 cubicweb/dataimport/importer.py --- a/cubicweb/dataimport/importer.py Thu Jan 28 14:02:31 2016 +0100 +++ b/cubicweb/dataimport/importer.py Thu Jan 28 18:18:44 2016 +0100 @@ -252,25 +252,31 @@ :class:`ExtEntity`, into CubicWeb entities. :param schema: the CubicWeb's instance schema + :param store: a CubicWeb `Store` + :param extid2eid: optional {extid: eid} dictionary giving information on existing entities. It will be completed during import. You may want to use :func:`cwuri2eid` to build it. - :param existing_relation: optional {rtype: set((subj eid, obj eid))} mapping giving information on - existing relations of a given type. You may want to use :class:`RelationMapping` to build it. - :param etypes_order_hint: optional ordered iterable on entity types, giving an hint on the order in - which they should be attempted to be imported - :param import_log: optional object implementing the :class:`SimpleImportLog` interface to record - events occuring during the import - :param raise_on_error: optional boolean flag - default to false, indicating whether errors should - be raised or logged. You usually want them to be raised during test but to be logged in - production. + + :param existing_relation: optional {rtype: set((subj eid, obj eid))} mapping giving information + on existing relations of a given type. You may want to use :class:`RelationMapping` to build + it. + + :param etypes_order_hint: optional ordered iterable on entity types, giving an hint on the + order in which they should be attempted to be imported + + :param import_log: optional object implementing the :class:`SimpleImportLog` interface to + record events occuring during the import + + :param raise_on_error: optional boolean flag - default to false, indicating whether errors + should be raised or logged. You usually want them to be raised during test but to be logged + in production. Instances of this class are meant to import external entities through :meth:`import_entities` which handles a stream of :class:`ExtEntity`. One may then plug arbitrary filters into the external entities stream. .. automethod:: import_entities - """ def __init__(self, schema, store, extid2eid=None, existing_relations=None, diff -r 3a83759854ee -r 682b15eb2dd2 cubicweb/dataimport/massive_store.py --- a/cubicweb/dataimport/massive_store.py Thu Jan 28 14:02:31 2016 +0100 +++ b/cubicweb/dataimport/massive_store.py Thu Jan 28 18:18:44 2016 +0100 @@ -28,13 +28,11 @@ from yams.constraints import SizeConstraint -from psycopg2 import ProgrammingError - +from cubicweb.schema import PURE_VIRTUAL_RTYPES from cubicweb.server.schema2sql import rschema_has_table -from cubicweb.schema import PURE_VIRTUAL_RTYPES +from cubicweb.server.sqlutils import SQL_PREFIX from cubicweb.dataimport import stores, pgstore from cubicweb.utils import make_uid -from cubicweb.server.sqlutils import SQL_PREFIX class MassiveObjectStore(stores.RQLObjectStore): @@ -138,7 +136,7 @@ cnx.read_security = False cnx.write_security = False - ### INIT FUNCTIONS ######################################################## + # INIT FUNCTIONS ######################################################## def _drop_all_constraints(self): schema = self._cnx.vreg.schema @@ -202,7 +200,7 @@ self._init_uri_eid.add(etype) - ### RELATE FUNCTION ####################################################### + # RELATE FUNCTION ####################################################### def relate_by_iid(self, iid_from, rtype, iid_to): """Add new relation based on the internal id (iid) @@ -214,7 +212,7 @@ iid_to = iid_to.encode('utf-8') self._data_uri_relations[rtype].append({'uri_from': iid_from, 'uri_to': iid_to}) - ### FLUSH FUNCTIONS ####################################################### + # FLUSH FUNCTIONS ####################################################### def flush_relations(self): """ Flush the relations data @@ -290,7 +288,7 @@ except Exception as ex: self.logger.error("Can't insert relation %s: %s", rtype, ex) - ### SQL UTILITIES ######################################################### + # SQL UTILITIES ######################################################### def drop_and_store_indexes(self, tablename): """Drop indexes and constraints""" @@ -375,7 +373,7 @@ 'entities_id_seq', initial_value=start_eid)) self._cnx.commit() - ### ENTITIES CREATION ##################################################### + # ENTITIES CREATION ##################################################### def _get_eid_gen(self): """ Function getting the next eid. This is done by preselecting @@ -471,7 +469,7 @@ self.sql('DROP TABLE IF EXISTS %s' % table_name) self.commit() - ### FLUSH ################################################################# + # FLUSH ################################################################# def on_commit(self): if self.on_commit_callback: @@ -600,8 +598,6 @@ % (rtype, eid_to, etype.lower())) -### CONSTRAINTS MANAGEMENT FUNCTIONS ########################################## - def get_size_constraints(schema): """analyzes yams ``schema`` and returns the list of size constraints. @@ -624,6 +620,7 @@ eschema_constraints[rschema.type] = maxsize return size_constraints + def get_default_values(schema): """analyzes yams ``schema`` and returns the list of default values. diff -r 3a83759854ee -r 682b15eb2dd2 cubicweb/dataimport/pgstore.py --- a/cubicweb/dataimport/pgstore.py Thu Jan 28 14:02:31 2016 +0100 +++ b/cubicweb/dataimport/pgstore.py Thu Jan 28 18:18:44 2016 +0100 @@ -225,7 +225,7 @@ self._add_relation = self._system_source.add_relation self.indexes_etypes = {} if nb_threads_statement != 1: - warn('[3.21] SQLGenObjectStore is no longer threaded', DeprecationWarning) + warnings.warn('[3.21] SQLGenObjectStore is no longer threaded', DeprecationWarning) def flush(self): """Flush data to the database""" diff -r 3a83759854ee -r 682b15eb2dd2 cubicweb/dataimport/test/data-massimport/schema.py --- a/cubicweb/dataimport/test/data-massimport/schema.py Thu Jan 28 14:02:31 2016 +0100 +++ b/cubicweb/dataimport/test/data-massimport/schema.py Thu Jan 28 18:18:44 2016 +0100 @@ -1,4 +1,4 @@ -# copyright 2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved. +# copyright 2015-2016 LOGILAB S.A. (Paris, FRANCE), all rights reserved. # contact http://www.logilab.fr -- mailto:contact@logilab.fr # # This program is free software: you can redistribute it and/or modify it under @@ -13,16 +13,14 @@ # # You should have received a copy of the GNU Lesser General Public License along # with this program. If not, see . +"""cubicweb-geonames schema -"""cubicweb-geonames schema""" +See geonames readme.txt for more details. +""" from yams.buildobjs import (EntityType, RelationDefinition, SubjectRelation, - String, Int, BigInt, Float, Date) -from cubicweb.schemas.base import ExternalUri + String, Int, BigInt, Float, Date) -""" -See geonames readme.txt for more details. -""" class TestLocation(EntityType): """ @@ -32,6 +30,7 @@ name = String(maxsize=1024, indexed=True, fulltextindexed=True) geonameid = Int(required=True, unique=True, indexed=True) + class Location(EntityType): """ Entity type for location of Geonames. @@ -48,8 +47,10 @@ feature_code = SubjectRelation('FeatureCode', cardinality='?*', inlined=True) country = SubjectRelation('Country', cardinality='?*', inlined=True) alternate_country_code = String(maxsize=60) - main_administrative_region = SubjectRelation('AdministrativeRegion', cardinality='?*', inlined=True) - second_administrative_region = SubjectRelation('AdministrativeRegion', cardinality='?*', inlined=True) + main_administrative_region = SubjectRelation('AdministrativeRegion', + cardinality='?*', inlined=True) + second_administrative_region = SubjectRelation('AdministrativeRegion', + cardinality='?*', inlined=True) admin_code_1 = String(maxsize=124) admin_code_2 = String(maxsize=124) admin_code_3 = String(maxsize=20) @@ -60,6 +61,7 @@ timezone = SubjectRelation('TimeZone', cardinality='?*', inlined=True) geonames_date = Date() + class LocationName(EntityType): """ Name of a Location @@ -68,6 +70,7 @@ language = SubjectRelation('Language', cardinality='?*', inlined=True) alternatenamesid = Int(indexed=True) + class FeatureCode(EntityType): """ Entity type for feature codes of Geonames. @@ -78,6 +81,7 @@ code = String(maxsize=12) description = String(maxsize=1024, fulltextindexed=True) + class AdministrativeRegion(EntityType): """ Entity type for administrative regions of Geonames. @@ -89,6 +93,7 @@ geonameid = Int(indexed=True) asciiname = String(maxsize=200, fulltextindexed=True) + class Language(EntityType): """ Entity type for languages of Geonames. @@ -99,6 +104,7 @@ iso_639_2 = String(maxsize=64, indexed=True) iso_639_1 = String(maxsize=3, indexed=True) + class Continent(EntityType): """ Entity type for continents of geonames. @@ -107,6 +113,7 @@ code = String(maxsize=2, indexed=True) geonameid = Int(indexed=True) + class Country(EntityType): """ Entity type for countries of geonames. @@ -133,6 +140,7 @@ neighbours_code = String(maxsize=200) equivalent_fips = String(maxsize=2) + class TimeZone(EntityType): """ Entity type for timezone of geonames. @@ -143,11 +151,13 @@ dst = Float() raw_offset = Float() + class used_language(RelationDefinition): subject = 'Country' object = 'Language' cardinality = '**' + class neighbour_of(RelationDefinition): subject = 'Country' object = 'Country'