--- a/cubicweb/dataimport/csv.py Thu Jan 28 14:02:31 2016 +0100
+++ b/cubicweb/dataimport/csv.py Thu Jan 28 18:18:44 2016 +0100
@@ -21,7 +21,6 @@
import codecs
import csv as csvmod
import warnings
-import os.path as osp
from six import PY2, PY3, string_types
@@ -37,7 +36,7 @@
for i, line in enumerate(f):
pass
f.seek(0)
- return i+1
+ return i + 1
def ucsvreader_pb(stream_or_path, encoding='utf-8', delimiter=',', quotechar='"',
--- a/cubicweb/dataimport/importer.py Thu Jan 28 14:02:31 2016 +0100
+++ b/cubicweb/dataimport/importer.py Thu Jan 28 18:18:44 2016 +0100
@@ -252,25 +252,31 @@
:class:`ExtEntity`, into CubicWeb entities.
:param schema: the CubicWeb's instance schema
+
:param store: a CubicWeb `Store`
+
:param extid2eid: optional {extid: eid} dictionary giving information on existing entities. It
will be completed during import. You may want to use :func:`cwuri2eid` to build it.
- :param existing_relation: optional {rtype: set((subj eid, obj eid))} mapping giving information on
- existing relations of a given type. You may want to use :class:`RelationMapping` to build it.
- :param etypes_order_hint: optional ordered iterable on entity types, giving an hint on the order in
- which they should be attempted to be imported
- :param import_log: optional object implementing the :class:`SimpleImportLog` interface to record
- events occuring during the import
- :param raise_on_error: optional boolean flag - default to false, indicating whether errors should
- be raised or logged. You usually want them to be raised during test but to be logged in
- production.
+
+ :param existing_relation: optional {rtype: set((subj eid, obj eid))} mapping giving information
+ on existing relations of a given type. You may want to use :class:`RelationMapping` to build
+ it.
+
+ :param etypes_order_hint: optional ordered iterable on entity types, giving an hint on the
+ order in which they should be attempted to be imported
+
+ :param import_log: optional object implementing the :class:`SimpleImportLog` interface to
+ record events occuring during the import
+
+ :param raise_on_error: optional boolean flag - default to false, indicating whether errors
+ should be raised or logged. You usually want them to be raised during test but to be logged
+ in production.
Instances of this class are meant to import external entities through :meth:`import_entities`
which handles a stream of :class:`ExtEntity`. One may then plug arbitrary filters into the
external entities stream.
.. automethod:: import_entities
-
"""
def __init__(self, schema, store, extid2eid=None, existing_relations=None,
--- a/cubicweb/dataimport/massive_store.py Thu Jan 28 14:02:31 2016 +0100
+++ b/cubicweb/dataimport/massive_store.py Thu Jan 28 18:18:44 2016 +0100
@@ -28,13 +28,11 @@
from yams.constraints import SizeConstraint
-from psycopg2 import ProgrammingError
-
+from cubicweb.schema import PURE_VIRTUAL_RTYPES
from cubicweb.server.schema2sql import rschema_has_table
-from cubicweb.schema import PURE_VIRTUAL_RTYPES
+from cubicweb.server.sqlutils import SQL_PREFIX
from cubicweb.dataimport import stores, pgstore
from cubicweb.utils import make_uid
-from cubicweb.server.sqlutils import SQL_PREFIX
class MassiveObjectStore(stores.RQLObjectStore):
@@ -138,7 +136,7 @@
cnx.read_security = False
cnx.write_security = False
- ### INIT FUNCTIONS ########################################################
+ # INIT FUNCTIONS ########################################################
def _drop_all_constraints(self):
schema = self._cnx.vreg.schema
@@ -202,7 +200,7 @@
self._init_uri_eid.add(etype)
- ### RELATE FUNCTION #######################################################
+ # RELATE FUNCTION #######################################################
def relate_by_iid(self, iid_from, rtype, iid_to):
"""Add new relation based on the internal id (iid)
@@ -214,7 +212,7 @@
iid_to = iid_to.encode('utf-8')
self._data_uri_relations[rtype].append({'uri_from': iid_from, 'uri_to': iid_to})
- ### FLUSH FUNCTIONS #######################################################
+ # FLUSH FUNCTIONS #######################################################
def flush_relations(self):
""" Flush the relations data
@@ -290,7 +288,7 @@
except Exception as ex:
self.logger.error("Can't insert relation %s: %s", rtype, ex)
- ### SQL UTILITIES #########################################################
+ # SQL UTILITIES #########################################################
def drop_and_store_indexes(self, tablename):
"""Drop indexes and constraints"""
@@ -375,7 +373,7 @@
'entities_id_seq', initial_value=start_eid))
self._cnx.commit()
- ### ENTITIES CREATION #####################################################
+ # ENTITIES CREATION #####################################################
def _get_eid_gen(self):
""" Function getting the next eid. This is done by preselecting
@@ -471,7 +469,7 @@
self.sql('DROP TABLE IF EXISTS %s' % table_name)
self.commit()
- ### FLUSH #################################################################
+ # FLUSH #################################################################
def on_commit(self):
if self.on_commit_callback:
@@ -600,8 +598,6 @@
% (rtype, eid_to, etype.lower()))
-### CONSTRAINTS MANAGEMENT FUNCTIONS ##########################################
-
def get_size_constraints(schema):
"""analyzes yams ``schema`` and returns the list of size constraints.
@@ -624,6 +620,7 @@
eschema_constraints[rschema.type] = maxsize
return size_constraints
+
def get_default_values(schema):
"""analyzes yams ``schema`` and returns the list of default values.
--- a/cubicweb/dataimport/pgstore.py Thu Jan 28 14:02:31 2016 +0100
+++ b/cubicweb/dataimport/pgstore.py Thu Jan 28 18:18:44 2016 +0100
@@ -225,7 +225,7 @@
self._add_relation = self._system_source.add_relation
self.indexes_etypes = {}
if nb_threads_statement != 1:
- warn('[3.21] SQLGenObjectStore is no longer threaded', DeprecationWarning)
+ warnings.warn('[3.21] SQLGenObjectStore is no longer threaded', DeprecationWarning)
def flush(self):
"""Flush data to the database"""
--- a/cubicweb/dataimport/test/data-massimport/schema.py Thu Jan 28 14:02:31 2016 +0100
+++ b/cubicweb/dataimport/test/data-massimport/schema.py Thu Jan 28 18:18:44 2016 +0100
@@ -1,4 +1,4 @@
-# copyright 2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# copyright 2015-2016 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
# contact http://www.logilab.fr -- mailto:contact@logilab.fr
#
# This program is free software: you can redistribute it and/or modify it under
@@ -13,16 +13,14 @@
#
# You should have received a copy of the GNU Lesser General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
+"""cubicweb-geonames schema
-"""cubicweb-geonames schema"""
+See geonames readme.txt for more details.
+"""
from yams.buildobjs import (EntityType, RelationDefinition, SubjectRelation,
- String, Int, BigInt, Float, Date)
-from cubicweb.schemas.base import ExternalUri
+ String, Int, BigInt, Float, Date)
-"""
-See geonames readme.txt for more details.
-"""
class TestLocation(EntityType):
"""
@@ -32,6 +30,7 @@
name = String(maxsize=1024, indexed=True, fulltextindexed=True)
geonameid = Int(required=True, unique=True, indexed=True)
+
class Location(EntityType):
"""
Entity type for location of Geonames.
@@ -48,8 +47,10 @@
feature_code = SubjectRelation('FeatureCode', cardinality='?*', inlined=True)
country = SubjectRelation('Country', cardinality='?*', inlined=True)
alternate_country_code = String(maxsize=60)
- main_administrative_region = SubjectRelation('AdministrativeRegion', cardinality='?*', inlined=True)
- second_administrative_region = SubjectRelation('AdministrativeRegion', cardinality='?*', inlined=True)
+ main_administrative_region = SubjectRelation('AdministrativeRegion',
+ cardinality='?*', inlined=True)
+ second_administrative_region = SubjectRelation('AdministrativeRegion',
+ cardinality='?*', inlined=True)
admin_code_1 = String(maxsize=124)
admin_code_2 = String(maxsize=124)
admin_code_3 = String(maxsize=20)
@@ -60,6 +61,7 @@
timezone = SubjectRelation('TimeZone', cardinality='?*', inlined=True)
geonames_date = Date()
+
class LocationName(EntityType):
"""
Name of a Location
@@ -68,6 +70,7 @@
language = SubjectRelation('Language', cardinality='?*', inlined=True)
alternatenamesid = Int(indexed=True)
+
class FeatureCode(EntityType):
"""
Entity type for feature codes of Geonames.
@@ -78,6 +81,7 @@
code = String(maxsize=12)
description = String(maxsize=1024, fulltextindexed=True)
+
class AdministrativeRegion(EntityType):
"""
Entity type for administrative regions of Geonames.
@@ -89,6 +93,7 @@
geonameid = Int(indexed=True)
asciiname = String(maxsize=200, fulltextindexed=True)
+
class Language(EntityType):
"""
Entity type for languages of Geonames.
@@ -99,6 +104,7 @@
iso_639_2 = String(maxsize=64, indexed=True)
iso_639_1 = String(maxsize=3, indexed=True)
+
class Continent(EntityType):
"""
Entity type for continents of geonames.
@@ -107,6 +113,7 @@
code = String(maxsize=2, indexed=True)
geonameid = Int(indexed=True)
+
class Country(EntityType):
"""
Entity type for countries of geonames.
@@ -133,6 +140,7 @@
neighbours_code = String(maxsize=200)
equivalent_fips = String(maxsize=2)
+
class TimeZone(EntityType):
"""
Entity type for timezone of geonames.
@@ -143,11 +151,13 @@
dst = Float()
raw_offset = Float()
+
class used_language(RelationDefinition):
subject = 'Country'
object = 'Language'
cardinality = '**'
+
class neighbour_of(RelationDefinition):
subject = 'Country'
object = 'Country'