cubicweb: comparison cubicweb/server/sources/native.py

equal deleted inserted replaced

-:9c2fbb872e91
+:5e5e224239c3
 ATTR_MAP = {}
 NONSYSTEM_ETYPES = set()
 NONSYSTEM_RELATIONS = set()
 class LogCursor(object):
 def __init__(self, cursor):
 self.cu = cursor
 def execute(self, query, args=None):
 def _undo_check_relation_target(tentity, rdef, role):
 """check linked entity has not been redirected for this relation"""
 card = rdef.role_cardinality(role)
 if card in '?1' and tentity.related(rdef.rtype, role):
-raise _UndoException(tentity._cw._(
+msg = tentity._cw._(
 "Can't restore %(role)s relation %(rtype)s to entity %(eid)s which "
 "is already linked using this relation.")
-% {'role': neg_role(role),
+raise _UndoException(msg % {'role': neg_role(role),
 'rtype': rdef.rtype,
 'eid': tentity.eid})
 def _undo_rel_info(cnx, subj, rtype, obj):
 entities = []
 for role, eid in (('subject', subj), ('object', obj)):
 try:
 entities.append(cnx.entity_from_eid(eid))
 except UnknownEid:
-raise _UndoException(cnx._(
+msg = cnx._(
 "Can't restore relation %(rtype)s, %(role)s entity %(eid)s"
 " doesn't exist anymore.")
-% {'role': cnx._(role),
+raise _UndoException(msg % {'role': cnx._(role),
 'rtype': cnx._(rtype),
 'eid': eid})
 sentity, oentity = entities
 try:
 rschema = cnx.vreg.schema.rschema(rtype)
 rdef = rschema.rdefs[(sentity.cw_etype, oentity.cw_etype)]
 except KeyError:
-raise _UndoException(cnx._(
+msg = cnx._(
 "Can't restore relation %(rtype)s between %(subj)s and "
 "%(obj)s, that relation does not exists anymore in the "
 "schema.")
-% {'rtype': cnx._(rtype),
+raise _UndoException(msg % {'rtype': cnx._(rtype),
 'subj': subj,
 'obj': obj})
 return sentity, oentity, rdef
 def _undo_has_later_transaction(cnx, eid):
 return cnx.system_sql('''\
 SELECT T.tx_uuid FROM transactions AS TREF, transactions AS T
 WHERE TREF.tx_uuid='%(txuuid)s' AND T.tx_uuid!='%(txuuid)s'
 """adapter for source using the native cubicweb schema (see below)
 """
 sqlgen_class = SQLGenerator
 options = (
 ('db-driver',
-{'type' : 'string',
+{'type': 'string',
 'default': 'postgres',
 # XXX use choice type
 'help': 'database driver (postgres, sqlite, sqlserver2005)',
 'group': 'native-source', 'level': 0,
 }),
 ('db-host',
-{'type' : 'string',
+{'type': 'string',
 'default': '',
 'help': 'database host',
 'group': 'native-source', 'level': 1,
 }),
 ('db-port',
-{'type' : 'string',
+{'type': 'string',
 'default': '',
 'help': 'database port',
 'group': 'native-source', 'level': 1,
 }),
 ('db-name',
-{'type' : 'string',
+{'type': 'string',
 'default': Method('default_instance_id'),
 'help': 'database name',
 'group': 'native-source', 'level': 0,
 }),
 ('db-namespace',
-{'type' : 'string',
+{'type': 'string',
 'default': '',
 'help': 'database namespace (schema) name',
 'group': 'native-source', 'level': 1,
 }),
 ('db-user',
-{'type' : 'string',
+{'type': 'string',
 'default': CubicWebNoAppConfiguration.mode == 'user' and getlogin() or 'cubicweb',
 'help': 'database user',
 'group': 'native-source', 'level': 0,
 }),
 ('db-password',
-{'type' : 'password',
+{'type': 'password',
 'default': '',
 'help': 'database password',
 'group': 'native-source', 'level': 0,
 }),
 ('db-encoding',
-{'type' : 'string',
+{'type': 'string',
 'default': 'utf8',
 'help': 'database encoding',
 'group': 'native-source', 'level': 1,
 }),
 ('db-extra-arguments',
-{'type' : 'string',
+{'type': 'string',
 'default': '',
 'help': 'set to "Trusted_Connection" if you are using SQLServer and '
 'want trusted authentication for the database connection',
 'group': 'native-source', 'level': 2,
 }),
 finally:
 self.open_source_connections()
 else:
 raise ValueError('Unknown format %r' % format)
 def restore(self, backupfile, confirm, drop, format='native'):
 """method called to restore a backup of source's data"""
 if self.repo.config.init_cnxset_pool:
 self.close_source_connections()
 try:
 raise ValueError('Unknown format %r' % format)
 finally:
 if self.repo.config.init_cnxset_pool:
 self.open_source_connections()
 def init(self, activated, source_entity):
 try:
 # test if 'asource' column exists
 query = self.dbhelper.sql_add_limit_offset('SELECT asource FROM entities', 1)
 source_entity._cw.system_sql(query)
-except Exception as ex:
+except Exception:
 self.eid_type_source = self.eid_type_source_pre_131
 super(NativeSQLSource, self).init(activated, source_entity)
 self.init_creating(source_entity._cw.cnxset)
 def shutdown(self):
 self.cache_hit, self.cache_miss, self.no_cache = 0, 0, 0
 self.schema = schema
 try:
 self._rql_sqlgen.schema = schema
 except AttributeError:
-pass # __init__
+pass  # __init__
 for authentifier in self.authentifiers:
 authentifier.set_schema(self.schema)
 clear_cache(self, 'need_fti_indexation')
 def support_entity(self, etype, write=False):
 """return true if the given entity's type is handled by this adapter
 if write is true, return true only if it's a RW support
 """
-return not etype in NONSYSTEM_ETYPES
+return etype not in NONSYSTEM_ETYPES
 def support_relation(self, rtype, write=False):
 """return true if the given relation's type is handled by this adapter
 if write is true, return true only if it's a RW support
 """
 if write:
-return not rtype in NONSYSTEM_RELATIONS
+return rtype not in NONSYSTEM_RELATIONS
 # due to current multi-sources implementation, the system source
 # can't claim not supporting a relation
-return True #not rtype == 'content_for'
+return True  #not rtype == 'content_for'
 @statsd_timeit
 def authenticate(self, cnx, login, **kwargs):
 """return CWUser eid for the given login and other authentication
 information found in kwargs, else raise `AuthenticationError`
 if attr in edited:
 handler = getattr(storage, 'entity_%s' % event)
 to_restore = handler(entity, attr)
 restore_values.append((entity, attr, to_restore))
 try:
-yield # 2/ execute the source's instructions
+yield  # 2/ execute the source's instructions
 finally:
 # 3/ restore original values
 for entity, attr, value in restore_values:
 entity.cw_edited.edited_attribute(attr, value)
 """delete an entity from the source"""
 with self._storage_handler(cnx, entity, 'deleted'):
 if cnx.ertype_supports_undo(entity.cw_etype):
 attrs = [SQL_PREFIX + r.type
 for r in entity.e_schema.subject_relations()
-if (r.final or r.inlined) and not r in VIRTUAL_RTYPES]
+if (r.final or r.inlined) and r not in VIRTUAL_RTYPES]
 changes = self._save_attrs(cnx, entity, attrs)
 self._record_tx_action(cnx, 'tx_entity_actions', u'D',
 etype=text_type(entity.cw_etype), eid=entity.eid,
 changes=self._binary(pickle.dumps(changes)))
 attrs = {'cw_eid': entity.eid}
 sql = self.sqlgen.delete(SQL_PREFIX + entity.cw_etype, attrs)
 self.doexec(cnx, sql, attrs)
 def add_relation(self, cnx, subject, rtype, object, inlined=False):
 """add a relation to the source"""
-self._add_relations(cnx,  rtype, [(subject, object)], inlined)
+self._add_relations(cnx, rtype, [(subject, object)], inlined)
 if cnx.ertype_supports_undo(rtype):
 self._record_tx_action(cnx, 'tx_relation_actions', u'A',
 eid_from=subject, rtype=text_type(rtype), eid_to=object)
-def add_relations(self, cnx,  rtype, subj_obj_list, inlined=False):
+def add_relations(self, cnx, rtype, subj_obj_list, inlined=False):
 """add a relations to the source"""
 self._add_relations(cnx, rtype, subj_obj_list, inlined)
 if cnx.ertype_supports_undo(rtype):
 for subject, object in subj_obj_list:
 self._record_tx_action(cnx, 'tx_relation_actions', u'A',
 sql = []
 if inlined is False:
 attrs = [{'eid_from': subject, 'eid_to': object}
 for subject, object in subj_obj_list]
 sql.append((self.sqlgen.insert('%s_relation' % rtype, attrs[0]), attrs))
-else: # used by data import
+else:  # used by data import
 etypes = {}
 for subject, object in subj_obj_list:
 etype = cnx.entity_metas(subject)['type']
 if etype in etypes:
 etypes[etype].append((subject, object))
 etypes[etype] = [(subject, object)]
 for subj_etype, subj_obj_list in etypes.items():
 attrs = [{'cw_eid': subject, SQL_PREFIX + rtype: object}
 for subject, object in subj_obj_list]
 sql.append((self.sqlgen.update(SQL_PREFIX + etype, attrs[0],
 ['cw_eid']),
 attrs))
 for statement, attrs in sql:
 self.doexecmany(cnx, statement, attrs)
 def delete_relation(self, cnx, subject, rtype, object):
 if inlined:
 table = SQL_PREFIX + cnx.entity_metas(subject)['type']
 column = SQL_PREFIX + rtype
 sql = 'UPDATE %s SET %s=NULL WHERE %seid=%%(eid)s' % (table, column,
 SQL_PREFIX)
-attrs = {'eid' : subject}
+attrs = {'eid': subject}
 else:
 attrs = {'eid_from': subject, 'eid_to': object}
 sql = self.sqlgen.delete('%s_relation' % rtype, attrs)
 self.doexec(cnx, sql, attrs)
 except Exception as ex:
 if self.repo.config.mode != 'test':
 # during test we get those message when trying to alter sqlite
 # db schema
 self.info("sql: %r\n args: %s\ndbms message: %r",
 query, args, ex.args[0])
 if rollback:
 try:
 cnx.cnxset.rollback()
 if self.repo.config.mode != 'test':
 self.debug('transaction has been rolled back')
 return res
 except Exception:
 self.exception('failed to query entities table for eid %s', eid)
 raise UnknownEid(eid)
-def eid_type_source(self, cnx, eid): # pylint: disable=E0202
+def eid_type_source(self, cnx, eid):  # pylint: disable=E0202
 """return a tuple (type, extid, source) for the entity with id <eid>"""
 sql = 'SELECT type, extid, asource FROM entities WHERE eid=%s' % eid
 res = self._eid_type_source(cnx, eid, sql)
 if not isinstance(res, list):
 res = list(res)
 'asource': text_type(source.uri)}
 self._handle_insert_entity_sql(cnx, self.sqlgen.insert('entities', attrs), attrs)
 # insert core relations: is, is_instance_of and cw_source
 if entity.e_schema.eid is not None:  # else schema has not yet been serialized
-self._handle_is_relation_sql(cnx, 'INSERT INTO is_relation(eid_from,eid_to) VALUES (%s,%s)',
+self._handle_is_relation_sql(
-(entity.eid, entity.e_schema.eid))
+cnx, 'INSERT INTO is_relation(eid_from,eid_to) VALUES (%s,%s)',
+(entity.eid, entity.e_schema.eid))
 for eschema in entity.e_schema.ancestors() + [entity.e_schema]:
-self._handle_is_relation_sql(cnx,
+self._handle_is_relation_sql(
-'INSERT INTO is_instance_of_relation(eid_from,eid_to) VALUES (%s,%s)',
+cnx,
-(entity.eid, eschema.eid))
+'INSERT INTO is_instance_of_relation(eid_from,eid_to) VALUES (%s,%s)',
+(entity.eid, eschema.eid))
 if source.eid is not None:  # else the source has not yet been inserted
-self._handle_is_relation_sql(cnx, 'INSERT INTO cw_source_relation(eid_from,eid_to) VALUES (%s,%s)',
+self._handle_is_relation_sql(
-(entity.eid, source.eid))
+cnx, 'INSERT INTO cw_source_relation(eid_from,eid_to) VALUES (%s,%s)',
+(entity.eid, source.eid))
 # now we can update the full text index
 if self.need_fti_indexation(entity.cw_etype):
 self.index_entity(cnx, entity=entity)
 def update_info(self, cnx, entity, need_fti_update):
 restr['tx_user'] = ueid
 sql = self.sqlgen.select('transactions', restr, ('tx_uuid', 'tx_time', 'tx_user'))
 if actionfilters:
 # we will need subqueries to filter transactions according to
 # actions done
-tearestr = {} # filters on the tx_entity_actions table
+tearestr = {}  # filters on the tx_entity_actions table
-trarestr = {} # filters on the tx_relation_actions table
+trarestr = {}  # filters on the tx_relation_actions table
-genrestr = {} # generic filters, appliyable to both table
+genrestr = {}  # generic filters, appliyable to both table
 # unless public explicitly set to false, we only consider public
 # actions
 if actionfilters.pop('public', True):
 genrestr['txa_public'] = True
 # put additional filters in trarestr and/or tearestr
 for key, val in actionfilters.items():
 if key == 'etype':
 # filtering on etype implies filtering on entity actions
 # only, and with no eid specified
 assert actionfilters.get('action', 'C') in 'CUD'
-assert not 'eid' in actionfilters
+assert 'eid' not in actionfilters
 tearestr['etype'] = text_type(val)
 elif key == 'eid':
 # eid filter may apply to 'eid' of tx_entity_actions or to
 # 'eid_from' OR 'eid_to' of tx_relation_actions
 if actionfilters.get('action', 'C') in 'CUD':
 sql = self.sqlgen.select('tx_entity_actions', restr,
 ('txa_action', 'txa_public', 'txa_order',
 'etype', 'eid', 'changes'))
 with cnx.ensure_cnx_set:
 cu = self.doexec(cnx, sql, restr)
-actions = [tx.EntityAction(a,p,o,et,e,c and pickle.loads(self.binary_to_str(c)))
+actions = [tx.EntityAction(a, p, o, et, e, c and pickle.loads(self.binary_to_str(c)))
-for a,p,o,et,e,c in cu.fetchall()]
+for a, p, o, et, e, c in cu.fetchall()]
 sql = self.sqlgen.select('tx_relation_actions', restr,
 ('txa_action', 'txa_public', 'txa_order',
 'rtype', 'eid_from', 'eid_to'))
 with cnx.ensure_cnx_set:
 cu = self.doexec(cnx, sql, restr)
 eschema = entity.e_schema
 getrschema = eschema.subjrels
 for column, value in changes.items():
 rtype = column[len(SQL_PREFIX):]
 if rtype == "eid":
-continue # XXX should even `eid` be stored in action changes?
+continue  # XXX should even `eid` be stored in action changes?
 try:
 rschema = getrschema[rtype]
 except KeyError:
 err(cnx._("can't restore relation %(rtype)s of entity %(eid)s, "
 "this relation does not exist in the schema anymore.")
 % {'rtype': rtype, 'eid': eid})
 if not rschema.final:
 if not rschema.inlined:
 assert value is None
 # rschema is an inlined relation
 elif value is not None:
 # not a deletion: we must put something in edited
 try:
-entity._cw.entity_from_eid(value) # check target exists
+entity._cw.entity_from_eid(value)  # check target exists
 edited[rtype] = value
 except UnknownEid:
 err(cnx._("can't restore entity %(eid)s of type %(eschema)s, "
 "target of %(rtype)s (eid %(value)s) does not exist any longer")
 % locals())
 changes[column] = None
 elif eschema.destination(rtype) in ('Bytes', 'Password'):
 changes[column] = self._binary(value)
 edited[rtype] = Binary(value)
 """undo an entity deletion"""
 errors = []
 err = errors.append
 eid = action.eid
 etype = action.etype
-_ = cnx._
 # get an entity instance
 try:
 entity = self.repo.vreg['etypes'].etype_class(etype)(cnx)
 except Exception:
 err("can't restore entity %s of type %s, type no more supported"
 eid = action.eid
 # XXX done to avoid fetching all remaining relation for the entity
 # we should find an efficient way to do this (keeping current veolidf
 # massive deletion performance)
 if _undo_has_later_transaction(cnx, eid):
-msg = cnx._('some later transaction(s) touch entity, undo them '
+msg = cnx._('some later transaction(s) touch entity, undo them first')
-'first')
 raise ValidationError(eid, {None: msg})
 etype = action.etype
 # get an entity instance
 try:
 entity = self.repo.vreg['etypes'].etype_class(etype)(cnx)
 err = errors.append
 try:
 entity = cnx.entity_from_eid(action.eid)
 except UnknownEid:
 err(cnx._("can't restore state of entity %s, it has been "
 "deleted inbetween") % action.eid)
 return errors
 self._reedit_entity(entity, action.changes, err)
 entity.cw_edited.check()
 self.repo.hm.call_hooks('before_update_entity', cnx, entity=entity)
 sql = self.sqlgen.update(SQL_PREFIX + entity.cw_etype, action.changes,
 cursor = cnx.cnxset.cu
 cursor_unindex_object = self.dbhelper.cursor_unindex_object
 try:
 for entity in entities:
 cursor_unindex_object(entity.eid, cursor)
-except Exception: # let KeyboardInterrupt / SystemExit propagate
+except Exception:  # let KeyboardInterrupt / SystemExit propagate
 self.exception('error while unindexing %s', entity)
 def fti_index_entities(self, cnx, entities):
 """add text content of created/modified entities to the full text index
 """
 cursor_index_object = self.dbhelper.cursor_index_object
 # unindexing done in the FTIndexEntityOp
 for entity in entities:
 cursor_index_object(entity.eid,
 entity.cw_adapt_to('IFTIndexable'),
 cursor)
-except Exception: # let KeyboardInterrupt / SystemExit propagate
+except Exception:  # let KeyboardInterrupt / SystemExit propagate
 self.exception('error while indexing %s', entity)
 class FTIndexEntityOp(hook.DataOperationMixIn, hook.LateOperation):
 """operation to delay entity full text indexation to commit
 done.add(eid)
 iftindexable = cnx.entity_from_eid(eid).cw_adapt_to('IFTIndexable')
 to_reindex |= set(iftindexable.fti_containers())
 source.fti_unindex_entities(cnx, to_reindex)
 source.fti_index_entities(cnx, to_reindex)
 def sql_schema(driver):
 """Yield SQL statements to create system tables in the database."""
 helper = get_db_helper(driver)
 typemap = helper.TYPE_MAPPING
 def set_schema(self, schema):
 """set the instance'schema"""
 pass
 class LoginPasswordAuthentifier(BaseAuthentifier):
 passwd_rql = 'Any P WHERE X is CWUser, X login %(login)s, X upassword P'
 auth_rql = (u'Any X WHERE X is CWUser, X login %(login)s, X upassword %(pwd)s, '
 'X cw_source S, S name "system"')
 _sols = ({'X': 'CWUser', 'P': 'Password', 'S': 'CWSource'},)
 def set_schema(self, schema):
 """set the instance'schema"""
-if 'CWUser' in schema: # probably an empty schema if not true...
+if 'CWUser' in schema:  # probably an empty schema if not true...
 # rql syntax trees used to authenticate users
 self._passwd_rqlst = self.source.compile_rql(self.passwd_rql, self._sols)
 self._auth_rqlst = self.source.compile_rql(self.auth_rql, self._sols)
 def authenticate(self, cnx, login, password=None, **kwargs):
 defined in this source, else raise `AuthenticationError`
 two queries are needed since passwords are stored crypted, so we have
 to fetch the salt first
 """
-args = {'login': login, 'pwd' : None}
+args = {'login': login, 'pwd': None}
 if password is not None:
 rset = self.source.syntax_tree_search(cnx, self._passwd_rqlst, args)
 try:
 pwd = rset[0][0]
 except IndexError:
 user = rset[0][0]
 # If the stored hash uses a deprecated scheme (e.g. DES or MD5 used
 # before 3.14.7), update with a fresh one
 if pwd is not None and pwd.getvalue():
 verify, newhash = verify_and_update(password, pwd.getvalue())
-if not verify: # should not happen, but...
+if not verify:  # should not happen, but...
 raise AuthenticationError('bad password')
 if newhash:
-cnx.system_sql("UPDATE %s SET %s=%%(newhash)s WHERE %s=%%(login)s" % (
+cnx.system_sql("UPDATE %s SET %s=%%(newhash)s WHERE %s=%%(login)s"
-SQL_PREFIX + 'CWUser',
+% (SQL_PREFIX + 'CWUser',
 SQL_PREFIX + 'upassword',
 SQL_PREFIX + 'login'),
 {'newhash': self.source._binary(newhash.encode('ascii')),
 'login': login})
 cnx.commit()
 return user
 except IndexError:
 raise AuthenticationError('bad password')
 class EmailPasswordAuthentifier(BaseAuthentifier):
 def authenticate(self, cnx, login, **authinfo):
 # email_auth flag prevent from infinite recursion (call to
 # repo.check_auth_info at the end of this method may lead us here again)
-if not '@' in login or authinfo.pop('email_auth', None):
+if '@' not in login or authinfo.pop('email_auth', None):
 raise AuthenticationError('not an email')
 rset = cnx.execute('Any L WHERE U login L, U primary_email M, '
 'M address %(login)s', {'login': login},
 build_descr=False)
 if rset.rowcount != 1:
 raise AuthenticationError('unexisting email')
 login = rset.rows[0][0]
 authinfo['email_auth'] = True
 return self.source.repo.check_auth_info(cnx, login, authinfo)
 prefix = 'cw_'
 for etype in self.schema.entities():
 eschema = self.schema.eschema(etype)
 if eschema.final:
 continue
-etype_tables.append('%s%s'%(prefix, etype))
+etype_tables.append('%s%s' % (prefix, etype))
 for rtype in self.schema.relations():
 rschema = self.schema.rschema(rtype)
 if rschema.final or rschema.inlined or rschema in VIRTUAL_RTYPES:
 continue
 relation_tables.append('%s_relation' % rtype)
 for i, start in enumerate(range(0, rowcount, blocksize)):
 rows = list(itertools.islice(rows_iterator, blocksize))
 serialized = self._serialize(table, columns, rows)
 archive.writestr('tables/%s.%04d' % (table, i), serialized)
 self.logger.debug('wrote rows %d to %d (out of %d) to %s.%04d',
-start, start+len(rows)-1,
+start, start + len(rows) - 1,
 rowcount,
 table, i)
 else:
 rows = []
 serialized = self._serialize(table, columns, rows)
 self.cursor.execute(insert, merge_args(dict(zip(columns, row)), {}))
 row_count += len(rows)
 self.cnx.commit()
 self.logger.info('inserted %d rows', row_count)
 def _parse_versions(self, version_str):
 versions = set()
 for line in version_str.splitlines():
 versions.add(tuple(line.split()))
 return versions

changeset 11417	5e5e224239c3
parent 11413	c172fa18565e
child 11429	6a9a9ea1e9b9