cubicweb/dataimport/stores.py
changeset 11309 31bf3254be69
parent 11308 df75fe529ba8
child 11749 ae9789d77ea0
equal deleted inserted replaced
11308:df75fe529ba8 11309:31bf3254be69
    52 
    52 
    53 * ``finish() -> None``: additional stuff to do after import is terminated.
    53 * ``finish() -> None``: additional stuff to do after import is terminated.
    54 
    54 
    55 .. autoclass:: cubicweb.dataimport.stores.RQLObjectStore
    55 .. autoclass:: cubicweb.dataimport.stores.RQLObjectStore
    56 .. autoclass:: cubicweb.dataimport.stores.NoHookRQLObjectStore
    56 .. autoclass:: cubicweb.dataimport.stores.NoHookRQLObjectStore
    57 .. autoclass:: cubicweb.dataimport.stores.MetaGenerator
    57 .. autoclass:: cubicweb.dataimport.stores.MetadataGenerator
    58 """
    58 """
    59 import inspect
    59 import inspect
    60 import warnings
    60 import warnings
    61 from datetime import datetime
    61 from datetime import datetime
    62 from copy import copy
    62 from copy import copy
    63 
    63 
    64 from six import text_type
    64 from six import text_type, add_metaclass
    65 
    65 
    66 import pytz
    66 import pytz
    67 
    67 
    68 from logilab.common.deprecation import deprecated
       
    69 from logilab.common.decorators import cached
    68 from logilab.common.decorators import cached
       
    69 from logilab.common.deprecation import deprecated, class_deprecated
    70 
    70 
    71 from cubicweb.schema import META_RTYPES, VIRTUAL_RTYPES
    71 from cubicweb.schema import META_RTYPES, VIRTUAL_RTYPES
    72 from cubicweb.server.edition import EditedEntity
    72 from cubicweb.server.edition import EditedEntity
    73 
    73 
    74 
    74 
   155     """Store that works by accessing low-level CubicWeb's source API, with all hooks deactivated. It
   155     """Store that works by accessing low-level CubicWeb's source API, with all hooks deactivated. It
   156     may be given a metadata generator object to handle metadata which are usually handled by hooks.
   156     may be given a metadata generator object to handle metadata which are usually handled by hooks.
   157 
   157 
   158     Arguments:
   158     Arguments:
   159     - `cnx`, a connection to the repository
   159     - `cnx`, a connection to the repository
   160     - `metagen`, optional :class:`MetaGenerator` instance
   160     - `metagen`, optional :class:`MetadataGenerator` instance
   161     """
   161     """
   162 
   162 
   163     def __init__(self, cnx, metagen=None):
   163     def __init__(self, cnx, metagen=None):
   164         super(NoHookRQLObjectStore, self).__init__(cnx)
   164         super(NoHookRQLObjectStore, self).__init__(cnx)
       
   165         if metagen is None:
       
   166             metagen = MetadataGenerator(cnx)
       
   167         if isinstance(metagen, MetadataGenerator):
       
   168             metagen = _MetaGeneratorBWCompatWrapper(metagen)
       
   169         self.metagen = metagen
   165         self._system_source = cnx.repo.system_source
   170         self._system_source = cnx.repo.system_source
   166         self._rschema = cnx.repo.schema.rschema
   171         self._rschema = cnx.repo.schema.rschema
   167         self._create_eid = cnx.repo.system_source.create_eid
   172         self._create_eid = self._system_source.create_eid
   168         self._add_relation = self.source.add_relation
   173         self._add_relation = self._system_source.add_relation
   169         if metagen is None:
       
   170             metagen = MetaGenerator(cnx)
       
   171         self.metagen = metagen
       
   172         self._nb_inserted_entities = 0
   174         self._nb_inserted_entities = 0
   173         self._nb_inserted_types = 0
   175         self._nb_inserted_types = 0
   174         self._nb_inserted_relations = 0
   176         self._nb_inserted_relations = 0
   175         # deactivate security
   177         # deactivate security
   176         cnx.read_security = False
   178         cnx.read_security = False
   237     @deprecated('[3.21] deprecated')
   239     @deprecated('[3.21] deprecated')
   238     def nb_inserted_relations(self):
   240     def nb_inserted_relations(self):
   239         return self._nb_inserted_relations
   241         return self._nb_inserted_relations
   240 
   242 
   241 
   243 
   242 class MetaGenerator(object):
   244 class MetadataGenerator(object):
   243     """Class responsible for generating standard metadata for imported entities. You may want to
   245     """Class responsible for generating standard metadata for imported entities. You may want to
   244     derive it to add application specific's metadata.
   246     derive it to add application specific's metadata. This class (or a subclass) may either be
       
   247     given to a nohook or massive store.
   245 
   248 
   246     Parameters:
   249     Parameters:
   247     * `cnx`: connection to the repository
   250     * `cnx`: connection to the repository
   248     * `baseurl`: optional base URL to be used for `cwuri` generation - default to config['base-url']
   251     * `baseurl`: optional base URL to be used for `cwuri` generation - default to config['base-url']
   249     * `source`: optional source to be used as `cw_source` for imported entities
   252     * `source`: optional source to be used as `cw_source` for imported entities
   250     """
   253     """
       
   254     META_RELATIONS = (META_RTYPES
       
   255                       - VIRTUAL_RTYPES
       
   256                       - set(('eid', 'cwuri',
       
   257                              'is', 'is_instance_of', 'cw_source')))
       
   258 
       
   259     def __init__(self, cnx, baseurl=None, source=None):
       
   260         self._cnx = cnx
       
   261         if baseurl is None:
       
   262             config = cnx.vreg.config
       
   263             baseurl = config['base-url'] or config.default_base_url()
       
   264         if not baseurl[-1] == '/':
       
   265             baseurl += '/'
       
   266         self._baseurl = baseurl
       
   267         if source is None:
       
   268             source = cnx.repo.system_source
       
   269         self.source = source
       
   270         self._need_extid = source is not cnx.repo.system_source
       
   271         self._now = datetime.now(pytz.utc)
       
   272         # attributes/relations shared by all entities of the same type
       
   273         self._etype_attrs = []
       
   274         self._etype_rels = []
       
   275         # attributes/relations specific to each entity
       
   276         self._entity_attrs = ['cwuri']
       
   277         rschema = cnx.vreg.schema.rschema
       
   278         for rtype in self.META_RELATIONS:
       
   279             # skip owned_by / created_by if user is the internal manager
       
   280             if cnx.user.eid == -1 and rtype in ('owned_by', 'created_by'):
       
   281                 continue
       
   282             if rschema(rtype).final:
       
   283                 self._etype_attrs.append(rtype)
       
   284             else:
       
   285                 self._etype_rels.append(rtype)
       
   286 
       
   287     # etype is provided in the 3 methods below as proven useful to custom implementation but not
       
   288     # used by the default implementation
       
   289 
       
   290     def etype_attrs(self, etype):
       
   291         """Return the list of attributes to be set for all entities of the given type."""
       
   292         return self._etype_attrs[:]
       
   293 
       
   294     def etype_rels(self, etype):
       
   295         """Return the list of relations to be set for all entities of the given type."""
       
   296         return self._etype_rels[:]
       
   297 
       
   298     def entity_attrs(self, etype):
       
   299         """Return the list of attributes whose value is set per instance, not per type, for the
       
   300         given type.
       
   301         """
       
   302         return self._entity_attrs[:]
       
   303 
       
   304     @cached
       
   305     def base_etype_attrs(self, etype):
       
   306         """Return a dictionary of attributes to be set for all entities of the given type."""
       
   307         attrs = {}
       
   308         for attr in self.etype_attrs(etype):
       
   309             genfunc = self._generator(attr)
       
   310             if genfunc:
       
   311                 attrs[attr] = genfunc(etype)
       
   312         return attrs
       
   313 
       
   314     @cached
       
   315     def base_etype_rels(self, etype):
       
   316         """Return a dictionary of relations to be set for all entities of the given type."""
       
   317         rels = {}
       
   318         for rel in self.etype_rels(etype):
       
   319             genfunc = self._generator(rel)
       
   320             if genfunc:
       
   321                 rels[rel] = genfunc(etype)
       
   322         return rels
       
   323 
       
   324     def entity_extid(self, etype, eid, attrs):
       
   325         """Return the extid for the entity of given type and eid, to be inserted in the 'entities'
       
   326         system table.
       
   327         """
       
   328         if self._need_extid:
       
   329             extid = attrs.get('cwuri')
       
   330             if extid is None:
       
   331                 raise Exception('entity from an external source but no extid specified')
       
   332             elif isinstance(extid, text_type):
       
   333                 extid = extid.encode('utf-8')
       
   334         else:
       
   335             extid = None
       
   336         return extid
       
   337 
       
   338     def init_entity_attrs(self, etype, eid, attrs):
       
   339         """Insert into an entity attrs dictionary attributes whose value is set per instance, not per
       
   340         type.
       
   341         """
       
   342         for attr in self.entity_attrs(etype):
       
   343             if attr in attrs:
       
   344                 # already set, skip this attribute
       
   345                 continue
       
   346             genfunc = self._generator(attr)
       
   347             if genfunc:
       
   348                 attrs[attr] = genfunc(etype, eid, attrs)
       
   349 
       
   350     def _generator(self, rtype):
       
   351         return getattr(self, 'gen_%s' % rtype, None)
       
   352 
       
   353     def gen_cwuri(self, etype, eid, attrs):
       
   354         assert self._baseurl, 'baseurl is None while generating cwuri'
       
   355         return u'%s%s' % (self._baseurl, eid)
       
   356 
       
   357     def gen_creation_date(self, etype):
       
   358         return self._now
       
   359 
       
   360     def gen_modification_date(self, etype):
       
   361         return self._now
       
   362 
       
   363     def gen_created_by(self, etype):
       
   364         return self._cnx.user.eid
       
   365 
       
   366     def gen_owned_by(self, etype):
       
   367         return self._cnx.user.eid
       
   368 
       
   369 
       
   370 class _MetaGeneratorBWCompatWrapper(object):
       
   371     """Class wrapping a MetadataGenerator to adapt it to the MetaGenerator interface.
       
   372     """
       
   373     META_RELATIONS = (META_RTYPES
       
   374                       - VIRTUAL_RTYPES
       
   375                       - set(('eid', 'cwuri',
       
   376                              'is', 'is_instance_of', 'cw_source')))
       
   377 
       
   378     def __init__(self, mdgenerator):
       
   379         self._mdgen = mdgenerator
       
   380 
       
   381     @cached
       
   382     def base_etype_dicts(self, etype):
       
   383         cnx = self._mdgen._cnx
       
   384         entity = cnx.vreg['etypes'].etype_class(etype)(cnx)
       
   385         # entity are "surface" copied, avoid shared dict between copies
       
   386         del entity.cw_extra_kwargs
       
   387         entity.cw_edited = EditedEntity(entity)
       
   388         attrs = self._mdgen.base_etype_attrs(etype)
       
   389         entity.cw_edited.update(attrs, skipsec=False)
       
   390         rels = self._mdgen.base_etype_rels(etype)
       
   391         return entity, rels
       
   392 
       
   393     def init_entity(self, entity):
       
   394         # if cwuri is specified, this is an extid. It's not if it's generated in the above loop
       
   395         extid = self._mdgen.entity_extid(entity.cw_etype, entity.eid, entity.cw_edited)
       
   396         attrs = dict(entity.cw_edited)
       
   397         self._mdgen.init_entity_attrs(entity.cw_etype, entity.eid, attrs)
       
   398         entity.cw_edited.update(attrs, skipsec=False)
       
   399         return self._mdgen.source, extid
       
   400 
       
   401 
       
   402 @add_metaclass(class_deprecated)
       
   403 class MetaGenerator(object):
       
   404     """Class responsible for generating standard metadata for imported entities. You may want to
       
   405     derive it to add application specific's metadata.
       
   406 
       
   407     Parameters:
       
   408     * `cnx`: connection to the repository
       
   409     * `baseurl`: optional base URL to be used for `cwuri` generation - default to config['base-url']
       
   410     * `source`: optional source to be used as `cw_source` for imported entities
       
   411     """
       
   412     __deprecation_warning__ = '[3.23] this class is deprecated, use MetadataGenerator instead'
       
   413 
   251     META_RELATIONS = (META_RTYPES
   414     META_RELATIONS = (META_RTYPES
   252                       - VIRTUAL_RTYPES
   415                       - VIRTUAL_RTYPES
   253                       - set(('eid', 'cwuri',
   416                       - set(('eid', 'cwuri',
   254                              'is', 'is_instance_of', 'cw_source')))
   417                              'is', 'is_instance_of', 'cw_source')))
   255 
   418