[repository] Drop the entities.extid column and associated cache
This was not necessary anymore with promoted usage of the new data import API.
Turn repository's _type_extid_cache to _type_cache with only the entity's type
as key.
This introduces an backward incompatible change: entity_metas dict doesn't
contains anymore the extid key, but it doesn't seem used at all anywhere, so
this sounds acceptable.
Closes #15538317
--- a/cubicweb/dataimport/massive_store.py Fri Sep 30 17:38:12 2016 +0200
+++ b/cubicweb/dataimport/massive_store.py Wed Oct 05 15:30:10 2016 +0200
@@ -146,7 +146,6 @@
self.sql('CREATE TABLE IF NOT EXISTS cwmassive_initialized'
'(retype text, type varchar(128))')
self.sql("INSERT INTO cwmassive_initialized VALUES (%(e)s, 'etype')", {'e': etype})
- self.sql('ALTER TABLE cw_%s ADD COLUMN extid VARCHAR(256)' % etype.lower())
attrs = self.metagen.base_etype_attrs(etype)
data = copy(attrs) # base_etype_attrs is @cached, a copy is necessary
data.update(kwargs)
@@ -158,10 +157,6 @@
default_values = self.default_values[etype]
missing_keys = set(default_values) - set(data)
data.update((key, default_values[key]) for key in missing_keys)
- extid = self.metagen.entity_extid(etype, data['eid'], data)
- if extid is not None:
- extid = b64encode(extid).decode('ascii')
- data['extid'] = extid
self.metagen.init_entity_attrs(etype, data['eid'], data)
self._data_entities[etype].append(data)
return data['eid']
@@ -203,9 +198,7 @@
cu = self.sql('SELECT retype, type FROM cwmassive_initialized')
for retype, _type in cu.fetchall():
self.logger.info('Cleanup for %s' % retype)
- if _type == 'etype':
- self.sql('ALTER TABLE cw_%s DROP COLUMN extid' % retype)
- elif _type == 'rtype':
+ if _type == 'rtype':
# Cleanup relations tables
self._cleanup_relations(retype)
self.sql('DELETE FROM cwmassive_initialized WHERE retype = %(e)s',
@@ -268,8 +261,7 @@
if not buf:
# The buffer is empty. This is probably due to error in _create_copyfrom_buffer
raise ValueError('Error in buffer creation for etype %s' % etype)
- columns = ['cw_%s' % attr if attr != 'extid' else attr
- for attr in columns]
+ columns = ['cw_%s' % attr for attr in columns]
cursor = self._cnx.cnxset.cu
try:
cursor.copy_from(buf, 'cw_%s' % etype.lower(), null='NULL', columns=columns)
@@ -303,8 +295,8 @@
for parent_eschema in chain(eschema.ancestors(), [eschema]):
self._insert_meta_relation(etype, parent_eschema.eid, 'is_instance_of_relation')
# finally insert records into the entities table
- self.sql("INSERT INTO entities (eid, type, extid) "
- "SELECT cw_eid, '%s', extid FROM cw_%s "
+ self.sql("INSERT INTO entities (eid, type) "
+ "SELECT cw_eid, '%s' FROM cw_%s "
"WHERE NOT EXISTS (SELECT 1 FROM entities WHERE eid=cw_eid)"
% (etype, etype.lower()))
--- a/cubicweb/dataimport/pgstore.py Fri Sep 30 17:38:12 2016 +0200
+++ b/cubicweb/dataimport/pgstore.py Wed Oct 05 15:30:10 2016 +0200
@@ -417,13 +417,10 @@
# add_info is _copypasted_ from the one in NativeSQLSource. We want it
# there because it will use the _handlers of the SQLGenSourceWrapper, which
# are not like the ones in the native source.
- def add_info(self, cnx, entity, source, extid):
+ def add_info(self, cnx, entity, source):
"""add type and source info for an eid into the system table"""
- # begin by inserting eid/type/source/extid into the entities table
- if extid is not None:
- assert isinstance(extid, binary_type)
- extid = b64encode(extid).decode('ascii')
- attrs = {'type': entity.cw_etype, 'eid': entity.eid, 'extid': extid}
+ # begin by inserting eid/type/source into the entities table
+ attrs = {'type': entity.cw_etype, 'eid': entity.eid}
self._handle_insert_entity_sql(cnx, self.sqlgen.insert('entities', attrs), attrs)
# insert core relations: is, is_instance_of and cw_source
self._handle_is_relation_sql(cnx, 'INSERT INTO is_relation(eid_from,eid_to) VALUES (%s,%s)',
--- a/cubicweb/dataimport/stores.py Fri Sep 30 17:38:12 2016 +0200
+++ b/cubicweb/dataimport/stores.py Wed Oct 05 15:30:10 2016 +0200
@@ -213,8 +213,8 @@
entity.cw_edited.update(kwargs, skipsec=False)
cnx = self._cnx
entity.eid = self._create_eid(cnx)
- entity_source, extid = self.metagen.init_entity(entity)
- self._system_source.add_info(cnx, entity, entity_source, extid)
+ entity_source = self.metagen.init_entity(entity)
+ self._system_source.add_info(cnx, entity, entity_source)
self._system_source.add_entity(cnx, entity)
kwargs = dict()
if inspect.getargspec(self._add_relation).keywords:
@@ -288,7 +288,6 @@
if source is None:
source = cnx.repo.system_source
self.source = source
- self._need_extid = source is not cnx.repo.system_source
self._now = datetime.now(pytz.utc)
# attributes/relations shared by all entities of the same type
self._etype_attrs = []
@@ -342,20 +341,6 @@
rels[rel] = genfunc(etype)
return rels
- def entity_extid(self, etype, eid, attrs):
- """Return the extid for the entity of given type and eid, to be inserted in the 'entities'
- system table.
- """
- if self._need_extid:
- extid = attrs.get('cwuri')
- if extid is None:
- raise Exception('entity from an external source but no extid specified')
- elif isinstance(extid, text_type):
- extid = extid.encode('utf-8')
- else:
- extid = None
- return extid
-
def init_entity_attrs(self, etype, eid, attrs):
"""Insert into an entity attrs dictionary attributes whose value is set per instance, not per
type.
@@ -412,12 +397,10 @@
return entity, rels
def init_entity(self, entity):
- # if cwuri is specified, this is an extid. It's not if it's generated in the above loop
- extid = self._mdgen.entity_extid(entity.cw_etype, entity.eid, entity.cw_edited)
attrs = dict(entity.cw_edited)
self._mdgen.init_entity_attrs(entity.cw_etype, entity.eid, attrs)
entity.cw_edited.update(attrs, skipsec=False)
- return self._mdgen.source, extid
+ return self._mdgen.source
@add_metaclass(class_deprecated)
@@ -482,10 +465,6 @@
return entity, rels
def init_entity(self, entity):
- # if cwuri is specified, this is an extid. It's not if it's generated in the above loop
- extid = entity.cw_edited.get('cwuri')
- if isinstance(extid, text_type):
- extid = extid.encode('utf-8')
for attr in self.entity_attrs:
if attr in entity.cw_edited:
# already set, skip this attribute
@@ -493,7 +472,7 @@
genfunc = self.generate(attr)
if genfunc:
entity.cw_edited.edited_attribute(attr, genfunc(entity))
- return self.source, extid
+ return self.source
def generate(self, rtype):
return getattr(self, 'gen_%s' % rtype, None)
--- a/cubicweb/dataimport/test/test_massive_store.py Fri Sep 30 17:38:12 2016 +0200
+++ b/cubicweb/dataimport/test/test_massive_store.py Wed Oct 05 15:30:10 2016 +0200
@@ -126,7 +126,6 @@
crs = cnx.system_sql('SELECT indexname FROM pg_indexes')
indexes = [r[0] for r in crs.fetchall()]
self.assertNotIn('entities_pkey', indexes)
- self.assertNotIn('entities_extid_idx', indexes)
self.assertNotIn('owned_by_relation_pkey', indexes)
self.assertNotIn('owned_by_relation_to_idx', indexes)
@@ -139,7 +138,6 @@
crs = cnx.system_sql('SELECT indexname FROM pg_indexes')
indexes = [r[0] for r in crs.fetchall()]
self.assertIn('entities_pkey', indexes)
- self.assertIn('entities_extid_idx', indexes)
self.assertIn(build_index_name('owned_by_relation', ['eid_from', 'eid_to'], 'key_'),
indexes)
self.assertIn(build_index_name('owned_by_relation', ['eid_from'], 'idx_'),
@@ -221,7 +219,6 @@
crs = cnx.system_sql('SELECT indexname FROM pg_indexes')
indexes = [r[0] for r in crs.fetchall()]
self.assertIn('entities_pkey', indexes)
- self.assertIn('entities_extid_idx', indexes)
self.assertIn(build_index_name('owned_by_relation', ['eid_from', 'eid_to'], 'key_'),
indexes)
self.assertIn(build_index_name('owned_by_relation', ['eid_from'], 'idx_'),
@@ -255,7 +252,6 @@
crs = cnx.system_sql('SELECT indexname FROM pg_indexes')
indexes = [r[0] for r in crs.fetchall()]
self.assertNotIn('entities_pkey', indexes)
- self.assertNotIn('entities_extid_idx', indexes)
self.assertNotIn(build_index_name('owned_by_relation', ['eid_from', 'eid_to'], 'key_'),
indexes)
self.assertNotIn(build_index_name('owned_by_relation', ['eid_from'], 'idx_'),
@@ -268,7 +264,6 @@
crs = cnx.system_sql('SELECT indexname FROM pg_indexes')
indexes = [r[0] for r in crs.fetchall()]
self.assertIn('entities_pkey', indexes)
- self.assertIn('entities_extid_idx', indexes)
self.assertIn(build_index_name('owned_by_relation', ['eid_from', 'eid_to'], 'key_'),
indexes)
self.assertIn(build_index_name('owned_by_relation', ['eid_from'], 'idx_'),
--- a/cubicweb/devtools/__init__.py Fri Sep 30 17:38:12 2016 +0200
+++ b/cubicweb/devtools/__init__.py Wed Oct 05 15:30:10 2016 +0200
@@ -125,7 +125,7 @@
if repo._needs_refresh:
for cnxset in repo.cnxsets:
cnxset.reconnect()
- repo._type_extid_cache = {}
+ repo._type_cache = {}
repo.querier._rql_cache = {}
repo.system_source.reset_caches()
repo._needs_refresh = False
--- a/cubicweb/devtools/repotest.py Fri Sep 30 17:38:12 2016 +0200
+++ b/cubicweb/devtools/repotest.py Wed Oct 05 15:30:10 2016 +0200
@@ -202,7 +202,7 @@
self._access = RepoAccess(self.repo, 'admin', FakeRequest)
self.ueid = self.session.user.eid
assert self.ueid != -1
- self.repo._type_extid_cache = {} # clear cache
+ self.repo._type_cache = {} # clear cache
self.maxeid = self.get_max_eid()
do_monkey_patch()
self._dumb_sessions = []
--- a/cubicweb/hooks/metadata.py Fri Sep 30 17:38:12 2016 +0200
+++ b/cubicweb/hooks/metadata.py Wed Oct 05 15:30:10 2016 +0200
@@ -147,55 +147,3 @@
elif ftcontainer == 'object':
cnx.repo.system_source.index_entity(
cnx, cnx.entity_from_eid(self.eidto))
-
-
-
-# entity source handling #######################################################
-
-class ChangeEntitySourceUpdateCaches(hook.Operation):
- oldsource = entity = None # make pylint happy
-
- def postcommit_event(self):
- self.oldsource.reset_caches()
- repo = self.cnx.repo
- entity = self.entity
- repo._type_extid_cache[entity.eid] = (entity.cw_etype, None)
-
-
-class ChangeEntitySourceDeleteHook(MetaDataHook):
- """support for moving an entity from an external source by watching 'Any
- cw_source CWSource' relation
- """
-
- __regid__ = 'cw.metadata.source-change'
- __select__ = MetaDataHook.__select__ & hook.match_rtype('cw_source')
- events = ('before_delete_relation',)
-
- def __call__(self):
- if (self._cw.deleted_in_transaction(self.eidfrom)
- or self._cw.deleted_in_transaction(self.eidto)):
- return
- schange = self._cw.transaction_data.setdefault('cw_source_change', {})
- schange[self.eidfrom] = self.eidto
-
-
-class ChangeEntitySourceAddHook(MetaDataHook):
- __regid__ = 'cw.metadata.source-change'
- __select__ = MetaDataHook.__select__ & hook.match_rtype('cw_source')
- events = ('before_add_relation',)
-
- def __call__(self):
- schange = self._cw.transaction_data.get('cw_source_change')
- if schange is not None and self.eidfrom in schange:
- newsource = self._cw.entity_from_eid(self.eidto)
- if newsource.name != 'system':
- raise Exception('changing source to something else than the '
- 'system source is unsupported')
- syssource = newsource.repo_source
- oldsource = self._cw.entity_from_eid(schange[self.eidfrom])
- entity = self._cw.entity_from_eid(self.eidfrom)
- attrs = {'type': entity.cw_etype, 'eid': entity.eid, 'extid': None}
- self._cw.system_sql(syssource.sqlgen.update('entities', attrs, ['eid']), attrs)
- # register an operation to update repository/sources caches
- ChangeEntitySourceUpdateCaches(self._cw, entity=entity,
- oldsource=oldsource.repo_source)
--- a/cubicweb/hooks/syncschema.py Fri Sep 30 17:38:12 2016 +0200
+++ b/cubicweb/hooks/syncschema.py Wed Oct 05 15:30:10 2016 +0200
@@ -309,9 +309,9 @@
self.info('renamed table %s to %s', oldname, newname)
sqlexec('UPDATE entities SET type=%(newname)s WHERE type=%(oldname)s',
{'newname': newname, 'oldname': oldname})
- for eid, (etype, extid) in cnx.repo._type_extid_cache.items():
+ for eid, etype in cnx.repo._type_cache.items():
if etype == oldname:
- cnx.repo._type_extid_cache[eid] = (newname, extid)
+ cnx.repo._type_cache[eid] = newname
# recreate the indexes
for rschema in eschema.subject_relations():
if rschema.inlined or (rschema.final and eschema.rdef(rschema.type).indexed):
--- a/cubicweb/misc/migration/3.21.0_Any.py Fri Sep 30 17:38:12 2016 +0200
+++ b/cubicweb/misc/migration/3.21.0_Any.py Wed Oct 05 15:30:10 2016 +0200
@@ -127,12 +127,6 @@
cu = session.cnxset.cu
helper = repo.system_source.dbhelper
-helper.drop_index(cu, 'entities', 'extid', False)
-# don't use create_index because it doesn't work for columns that may be NULL
-# on sqlserver
-for query in helper.sqls_create_multicol_unique_index('entities', ['extid']):
- cu.execute(query)
-
sql('DELETE FROM entities WHERE eid < 0')
commit()
--- a/cubicweb/misc/migration/3.24.0_Any.py Fri Sep 30 17:38:12 2016 +0200
+++ b/cubicweb/misc/migration/3.24.0_Any.py Wed Oct 05 15:30:10 2016 +0200
@@ -5,3 +5,4 @@
sql('DROP TABLE moved_entities')
sql('ALTER TABLE entities DROP COLUMN asource')
+sql('ALTER TABLE entities DROP COLUMN extid')
--- a/cubicweb/misc/migration/bootstrapmigration_repository.py Fri Sep 30 17:38:12 2016 +0200
+++ b/cubicweb/misc/migration/bootstrapmigration_repository.py Wed Oct 05 15:30:10 2016 +0200
@@ -379,15 +379,6 @@
sync_schema_props_perms()
-if applcubicwebversion < (3, 2, 2) and cubicwebversion >= (3, 2, 1):
- from base64 import b64encode
- for eid, extid in sql('SELECT eid, extid FROM entities '
- 'WHERE extid is NOT NULL',
- ask_confirm=False):
- sql('UPDATE entities SET extid=%(extid)s WHERE eid=%(eid)s',
- {'extid': b64encode(extid), 'eid': eid}, ask_confirm=False)
- commit()
-
if applcubicwebversion < (3, 2, 0) and cubicwebversion >= (3, 2, 0):
add_cube('card', update_database=False)
--- a/cubicweb/misc/scripts/ldap_change_base_dn.py Fri Sep 30 17:38:12 2016 +0200
+++ b/cubicweb/misc/scripts/ldap_change_base_dn.py Wed Oct 05 15:30:10 2016 +0200
@@ -14,12 +14,12 @@
raw_input("Ensure you've stopped the instance, type enter when done.")
-for eid, extid in sql("SELECT eid, extid FROM entities WHERE source='%s'" % uri):
- olduserdn = b64decode(extid)
+for eid, olduserdn in rql("Any X, XURI WHERE X cwuri XURI, X cw_source S, S name %(name)s",
+ {'name': uri}):
newuserdn = olduserdn.replace(olddn, newdn)
if newuserdn != olduserdn:
print(olduserdn, '->', newuserdn)
- sql("UPDATE entities SET extid='%s' WHERE eid=%s" % (b64encode(newuserdn), eid))
+ sql("UPDATE cw_cwuser SET cw_cwuri='%s' WHERE cw_eid=%s" % (newuserdn, eid))
commit()
--- a/cubicweb/server/checkintegrity.py Fri Sep 30 17:38:12 2016 +0200
+++ b/cubicweb/server/checkintegrity.py Wed Oct 05 15:30:10 2016 +0200
@@ -414,7 +414,6 @@
SYSTEM_INDICES = {
# see cw/server/sources/native.py
'entities_type_idx': ('entities', 'type'),
- 'entities_extid_idx': ('entities', 'extid'),
'transactions_tx_time_idx': ('transactions', 'tx_time'),
'transactions_tx_user_idx': ('transactions', 'tx_user'),
'tx_entity_actions_txa_action_idx': ('tx_entity_actions', 'txa_action'),
--- a/cubicweb/server/repository.py Fri Sep 30 17:38:12 2016 +0200
+++ b/cubicweb/server/repository.py Wed Oct 05 15:30:10 2016 +0200
@@ -179,9 +179,8 @@
self.sources_by_uri = {'system': self.system_source}
# querier helper, need to be created after sources initialization
self.querier = querier.QuerierHelper(self, self.schema)
- # cache eid -> (type, extid)
- self._type_extid_cache = {}
- # cache extid -> eid
+ # cache eid -> type
+ self._type_cache = {}
# open some connection sets
if config.init_cnxset_pool:
self.init_cnxset_pool()
@@ -714,33 +713,29 @@
# * correspondance between eid and local id (i.e. specific to a given source)
def clear_caches(self, eids):
- etcache = self._type_extid_cache
+ etcache = self._type_cache
rqlcache = self.querier._rql_cache
for eid in eids:
try:
- etype, extid = etcache.pop(int(eid)) # may be a string in some cases
+ etype = etcache.pop(int(eid)) # may be a string in some cases
rqlcache.pop(('%s X WHERE X eid %s' % (etype, eid),), None)
except KeyError:
etype = None
rqlcache.pop(('Any X WHERE X eid %s' % eid,), None)
self.system_source.clear_eid_cache(eid, etype)
- def type_and_extid_from_eid(self, eid, cnx):
- """Return the type and extid of the entity with id `eid`."""
+ def type_from_eid(self, eid, cnx):
+ """Return the type of the entity with id `eid`"""
try:
eid = int(eid)
except ValueError:
raise UnknownEid(eid)
try:
- return self._type_extid_cache[eid]
+ return self._type_cache[eid]
except KeyError:
- etype, extid = self.system_source.eid_type_extid(cnx, eid)
- self._type_extid_cache[eid] = (etype, extid)
- return etype, extid
-
- def type_from_eid(self, eid, cnx):
- """Return the type of the entity with id `eid`"""
- return self.type_and_extid_from_eid(eid, cnx)[0]
+ etype = self.system_source.eid_type(cnx, eid)
+ self._type_cache[eid] = etype
+ return etype
def querier_cache_key(self, cnx, rql, args, eidkeys):
cachekey = [rql]
@@ -757,13 +752,13 @@
args[key] = int(args[key])
return tuple(cachekey)
- def add_info(self, cnx, entity, source, extid=None):
+ def add_info(self, cnx, entity, source):
"""add type and source info for an eid into the system table,
and index the entity with the full text index
"""
- # begin by inserting eid/type/source/extid into the entities table
+ # begin by inserting eid/type/source into the entities table
hook.CleanupNewEidsCacheOp.get_instance(cnx).add_data(entity.eid)
- self.system_source.add_info(cnx, entity, source, extid)
+ self.system_source.add_info(cnx, entity, source)
def _delete_cascade_multi(self, cnx, entities):
"""same as _delete_cascade but accepts a list of entities with
@@ -804,16 +799,9 @@
entities, rql)
def init_entity_caches(self, cnx, entity, source):
- """add entity to connection entities cache and repo's extid cache.
- Return entity's ext id if the source isn't the system source.
- """
+ """Add entity to connection entities cache and repo's cache."""
cnx.set_entity_cache(entity)
- if source.uri == 'system':
- extid = None
- else:
- extid = source.get_extid(entity)
- self._type_extid_cache[entity.eid] = (entity.cw_etype, extid)
- return extid
+ self._type_cache[entity.eid] = entity.cw_etype
def glob_add_entity(self, cnx, edited):
"""add an entity to the repository
@@ -829,7 +817,7 @@
# allocate an eid to the entity before calling hooks
entity.eid = self.system_source.create_eid(cnx)
# set caches asap
- extid = self.init_entity_caches(cnx, entity, source)
+ self.init_entity_caches(cnx, entity, source)
if server.DEBUG & server.DBG_REPO:
print('ADD entity', self, entity.cw_etype, entity.eid, edited)
prefill_entity_caches(entity)
@@ -838,7 +826,7 @@
edited.set_defaults()
if cnx.is_hook_category_activated('integrity'):
edited.check(creation=True)
- self.add_info(cnx, entity, source, extid)
+ self.add_info(cnx, entity, source)
try:
source.add_entity(cnx, entity)
except (UniqueTogetherError, ViolatedConstraint) as exc:
--- a/cubicweb/server/schemaserial.py Fri Sep 30 17:38:12 2016 +0200
+++ b/cubicweb/server/schemaserial.py Wed Oct 05 15:30:10 2016 +0200
@@ -45,7 +45,7 @@
"""
res = {}
for eid, name in cnx.execute('Any G, N WHERE G is CWGroup, G name N',
- build_descr=False):
+ build_descr=False):
res[name] = eid
if not interactive:
return res
@@ -147,8 +147,8 @@
{'x': etype, 'n': netype})
cnx.commit(False)
tocleanup = [eid]
- tocleanup += (eid for eid, cached in repo._type_extid_cache.items()
- if etype == cached[0])
+ tocleanup += (eid for eid, cached in repo._type_cache.items()
+ if etype == cached)
repo.clear_caches(tocleanup)
cnx.commit(False)
if needcopy:
--- a/cubicweb/server/session.py Fri Sep 30 17:38:12 2016 +0200
+++ b/cubicweb/server/session.py Wed Oct 05 15:30:10 2016 +0200
@@ -779,10 +779,15 @@
return self.repo.source_defs()
@_open_only
+ def entity_type(self, eid):
+ """Return entity type for the entity with id `eid`."""
+ return self.repo.type_from_eid(eid, self)
+
+ @deprecated('[3.24] use entity_type(eid) instead')
+ @_open_only
def entity_metas(self, eid):
- """Return a dictionary {type, extid}) for the entity with id `eid`."""
- etype, extid = self.repo.type_and_extid_from_eid(eid, self)
- return {'type': etype, 'extid': extid}
+ """Return a dictionary {type}) for the entity with id `eid`."""
+ return {'type': self.repo.type_from_eid(eid, self)}
# core method #############################################################
--- a/cubicweb/server/sources/__init__.py Fri Sep 30 17:38:12 2016 +0200
+++ b/cubicweb/server/sources/__init__.py Wed Oct 05 15:30:10 2016 +0200
@@ -192,12 +192,6 @@
else:
self.urls = []
- @staticmethod
- def decode_extid(extid):
- if extid is None:
- return extid
- return b64decode(extid)
-
# source initialization / finalization #####################################
def set_schema(self, schema):
@@ -301,10 +295,6 @@
# write modification api ###################################################
# read-only sources don't have to implement methods below
- def get_extid(self, entity):
- """return the external id for the given newly inserted entity"""
- raise NotImplementedError(self)
-
def add_entity(self, cnx, entity):
"""add a new entity to the source"""
raise NotImplementedError(self)
@@ -339,14 +329,14 @@
# system source interface #################################################
- def eid_type_extid(self, cnx, eid):
- """return a tuple (type, extid) for the entity with id <eid>"""
+ def eid_type(self, cnx, eid):
+ """Return the type of entity `eid`."""
raise NotImplementedError(self)
def create_eid(self, cnx):
raise NotImplementedError(self)
- def add_info(self, cnx, entity, source, extid):
+ def add_info(self, cnx, entity, source):
"""add type and source info for an eid into the system table"""
raise NotImplementedError(self)
--- a/cubicweb/server/sources/native.py Fri Sep 30 17:38:12 2016 +0200
+++ b/cubicweb/server/sources/native.py Wed Oct 05 15:30:10 2016 +0200
@@ -15,14 +15,8 @@
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
-"""Adapters for native cubicweb sources.
+"""Adapters for native cubicweb sources."""
-Notes:
-* extid (aka external id, the primary key of an entity in the external source
- from which it comes from) are stored in a varchar column encoded as a base64
- string. This is because it should actually be Bytes but we want an index on
- it for fast querying.
-"""
from __future__ import print_function
from threading import Lock
@@ -814,16 +808,13 @@
# system source interface #################################################
- def eid_type_extid(self, cnx, eid): # pylint: disable=E0202
- """return a tuple (type, extid) for the entity with id <eid>"""
- sql = 'SELECT type, extid FROM entities WHERE eid=%s' % eid
+ def eid_type(self, cnx, eid): # pylint: disable=E0202
+ """Return the entity's type for `eid`."""
+ sql = 'SELECT type FROM entities WHERE eid=%s' % eid
try:
res = self.doexec(cnx, sql).fetchone()
if res is not None:
- if not isinstance(res, list):
- res = list(res)
- res[-1] = self.decode_extid(res[-1])
- return res
+ return res[0]
except Exception:
self.exception('failed to query entities table for eid %s', eid)
raise UnknownEid(eid)
@@ -836,14 +827,11 @@
_handle_insert_entity_sql = doexec
_handle_is_instance_of_sql = _handle_source_relation_sql = _handle_is_relation_sql
- def add_info(self, cnx, entity, source, extid):
+ def add_info(self, cnx, entity, source):
"""add type and source info for an eid into the system table"""
assert cnx.cnxset is not None
- # begin by inserting eid/type/source/extid into the entities table
- if extid is not None:
- assert isinstance(extid, binary_type)
- extid = b64encode(extid).decode('ascii')
- attrs = {'type': text_type(entity.cw_etype), 'eid': entity.eid, 'extid': extid}
+ # begin by inserting eid/type/source into the entities table
+ attrs = {'type': text_type(entity.cw_etype), 'eid': entity.eid}
self._handle_insert_entity_sql(cnx, self.sqlgen.insert('entities', attrs), attrs)
# insert core relations: is, is_instance_of and cw_source
@@ -1131,7 +1119,7 @@
# restore the entity
action.changes['cw_eid'] = eid
# restore record in entities (will update fti if needed)
- self.add_info(cnx, entity, self, None)
+ self.add_info(cnx, entity, self)
sql = self.sqlgen.insert(SQL_PREFIX + etype, action.changes)
self.doexec(cnx, sql, action.changes)
self.repo.hm.call_hooks('after_add_entity', cnx, entity=entity)
@@ -1332,8 +1320,7 @@
for sql in ("""
CREATE TABLE entities (
eid INTEGER PRIMARY KEY NOT NULL,
- type VARCHAR(64) NOT NULL,
- extid VARCHAR(256)
+ type VARCHAR(64) NOT NULL
);;
CREATE INDEX entities_type_idx ON entities(type);;
@@ -1387,11 +1374,6 @@
DELETE FROM tx_relation_actions WHERE tx_uuid=OLD.tx_uuid;
END;
'''
- # define a multi-columns index on a single index to please sqlserver, which doesn't like several
- # null entries in a UNIQUE column
- for sql in helper.sqls_create_multicol_unique_index('entities', ['extid'],
- 'entities_extid_idx'):
- yield sql
def grant_schema(user, set_owner=True):
--- a/cubicweb/server/test/unittest_datafeed.py Fri Sep 30 17:38:12 2016 +0200
+++ b/cubicweb/server/test/unittest_datafeed.py Wed Oct 05 15:30:10 2016 +0200
@@ -85,10 +85,7 @@
self.assertEqual(entity.cwuri, 'http://www.cubicweb.org/')
self.assertEqual(entity.cw_source[0].name, u'รด myfeed')
# test repo cache keys
- self.assertEqual(self.repo._type_extid_cache[entity.eid],
- ('Card', b'http://www.cubicweb.org/'))
- self.assertEqual(self.repo._type_extid_cache[entity.eid],
- ('Card', b'http://www.cubicweb.org/'))
+ self.assertEqual(self.repo._type_cache[entity.eid], 'Card')
self.assertTrue(dfsource.latest_retrieval)
self.assertTrue(dfsource.fresh())
--- a/cubicweb/server/test/unittest_querier.py Fri Sep 30 17:38:12 2016 +0200
+++ b/cubicweb/server/test/unittest_querier.py Wed Oct 05 15:30:10 2016 +0200
@@ -614,15 +614,15 @@
[[u'description_format', 13],
[u'description', 14],
[u'name', 19],
- [u'created_by', 46],
- [u'creation_date', 46],
- [u'cw_source', 46],
- [u'cwuri', 46],
- [u'in_basket', 46],
- [u'is', 46],
- [u'is_instance_of', 46],
- [u'modification_date', 46],
- [u'owned_by', 46]])
+ [u'created_by', 45],
+ [u'creation_date', 45],
+ [u'cw_source', 45],
+ [u'cwuri', 45],
+ [u'in_basket', 45],
+ [u'is', 45],
+ [u'is_instance_of', 45],
+ [u'modification_date', 45],
+ [u'owned_by', 45]])
def test_select_aggregat_having_dumb(self):
# dumb but should not raise an error
--- a/cubicweb/server/test/unittest_repository.py Fri Sep 30 17:38:12 2016 +0200
+++ b/cubicweb/server/test/unittest_repository.py Wed Oct 05 15:30:10 2016 +0200
@@ -442,7 +442,7 @@
self.repo.add_info(cnx, entity, self.repo.system_source)
cu = cnx.system_sql('SELECT * FROM entities WHERE eid = -1')
data = cu.fetchall()
- self.assertEqual(tuplify(data), [(-1, 'Personne', None)])
+ self.assertEqual(tuplify(data), [(-1, 'Personne')])
self.repo._delete_cascade_multi(cnx, [entity])
self.repo.system_source.delete_info_multi(cnx, [entity])
cu = cnx.system_sql('SELECT * FROM entities WHERE eid = -1')
--- a/cubicweb/sobjects/services.py Fri Sep 30 17:38:12 2016 +0200
+++ b/cubicweb/sobjects/services.py Wed Oct 05 15:30:10 2016 +0200
@@ -48,7 +48,7 @@
results['%s_cache_hit' % title] = hits
results['%s_cache_miss' % title] = misses
results['%s_cache_hit_percent' % title] = (hits * 100) / (hits + misses)
- results['type_extid_cache_size'] = len(repo._type_extid_cache)
+ results['type_cache_size'] = len(repo._type_cache)
results['sql_no_cache'] = repo.system_source.no_cache
results['nb_open_sessions'] = len(repo._sessions)
results['nb_active_threads'] = threading.activeCount()
--- a/cubicweb/web/views/debug.py Fri Sep 30 17:38:12 2016 +0200
+++ b/cubicweb/web/views/debug.py Wed Oct 05 15:30:10 2016 +0200
@@ -94,7 +94,7 @@
stats['looping_tasks'] = ', '.join('%s (%s seconds)' % (n, i) for n, i in stats['looping_tasks'])
stats['threads'] = ', '.join(sorted(stats['threads']))
for k in stats:
- if k == 'type_extid_cache_size':
+ if k == 'type_cache_size':
continue
if k.endswith('_cache_size'):
stats[k] = '%s / %s' % (stats[k]['size'], stats[k]['maxsize'])