# HG changeset patch # User Sylvain Thénault # Date 1297099176 -3600 # Node ID 0cf10429ad391e42c8fa0e492f9f89f13c945308 # Parent 406a41c25e135bd3b5192f6f2ba3e994d0e27804 [sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml) diff -r 406a41c25e13 -r 0cf10429ad39 entities/schemaobjs.py --- a/entities/schemaobjs.py Mon Feb 07 15:13:05 2011 +0100 +++ b/entities/schemaobjs.py Mon Feb 07 18:19:36 2011 +0100 @@ -1,4 +1,4 @@ -# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved. +# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved. # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr # # This file is part of CubicWeb. @@ -19,12 +19,7 @@ __docformat__ = "restructuredtext en" -import re -from socket import gethostname - from logilab.common.decorators import cached -from logilab.common.textutils import text_to_dict -from logilab.common.configuration import OptionError from yams.schema import role_name @@ -34,58 +29,6 @@ from cubicweb.entities import AnyEntity, fetch_config -class _CWSourceCfgMixIn(object): - @property - def dictconfig(self): - return self.config and text_to_dict(self.config) or {} - - def update_config(self, skip_unknown=False, **config): - from cubicweb.server import SOURCE_TYPES - from cubicweb.server.serverconfig import (SourceConfiguration, - generate_source_config) - cfg = self.dictconfig - cfg.update(config) - options = SOURCE_TYPES[self.type].options - sconfig = SourceConfiguration(self._cw.vreg.config, options=options) - for opt, val in cfg.iteritems(): - try: - sconfig.set_option(opt, val) - except OptionError: - if skip_unknown: - continue - raise - cfgstr = unicode(generate_source_config(sconfig), self._cw.encoding) - self.set_attributes(config=cfgstr) - - -class CWSource(_CWSourceCfgMixIn, AnyEntity): - __regid__ = 'CWSource' - fetch_attrs, fetch_order = fetch_config(['name', 'type']) - - @property - def host_config(self): - dictconfig = self.dictconfig - host = gethostname() - for hostcfg in self.host_configs: - if hostcfg.match(host): - self.info('matching host config %s for source %s', - hostcfg.match_host, self.name) - dictconfig.update(hostcfg.dictconfig) - return dictconfig - - @property - def host_configs(self): - return self.reverse_cw_host_config_of - - -class CWSourceHostConfig(_CWSourceCfgMixIn, AnyEntity): - __regid__ = 'CWSourceHostConfig' - fetch_attrs, fetch_order = fetch_config(['match_host', 'config']) - - def match(self, hostname): - return re.match(self.match_host, hostname) - - class CWEType(AnyEntity): __regid__ = 'CWEType' fetch_attrs, fetch_order = fetch_config(['name']) diff -r 406a41c25e13 -r 0cf10429ad39 entities/sources.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/entities/sources.py Mon Feb 07 18:19:36 2011 +0100 @@ -0,0 +1,125 @@ +# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved. +# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr +# +# This file is part of CubicWeb. +# +# CubicWeb is free software: you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation, either version 2.1 of the License, or (at your option) +# any later version. +# +# CubicWeb is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License along +# with CubicWeb. If not, see . +"""data source related entities""" + +__docformat__ = "restructuredtext en" + +import re +from socket import gethostname + +from logilab.common.textutils import text_to_dict +from logilab.common.configuration import OptionError + +from cubicweb import ValidationError +from cubicweb.entities import AnyEntity, fetch_config + +class _CWSourceCfgMixIn(object): + @property + def dictconfig(self): + return self.config and text_to_dict(self.config) or {} + + def update_config(self, skip_unknown=False, **config): + from cubicweb.server import SOURCE_TYPES + from cubicweb.server.serverconfig import (SourceConfiguration, + generate_source_config) + cfg = self.dictconfig + cfg.update(config) + options = SOURCE_TYPES[self.type].options + sconfig = SourceConfiguration(self._cw.vreg.config, options=options) + for opt, val in cfg.iteritems(): + try: + sconfig.set_option(opt, val) + except OptionError: + if skip_unknown: + continue + raise + cfgstr = unicode(generate_source_config(sconfig), self._cw.encoding) + self.set_attributes(config=cfgstr) + + +class CWSource(_CWSourceCfgMixIn, AnyEntity): + __regid__ = 'CWSource' + fetch_attrs, fetch_order = fetch_config(['name', 'type']) + + @property + def host_config(self): + dictconfig = self.dictconfig + host = gethostname() + for hostcfg in self.host_configs: + if hostcfg.match(host): + self.info('matching host config %s for source %s', + hostcfg.match_host, self.name) + dictconfig.update(hostcfg.dictconfig) + return dictconfig + + @property + def host_configs(self): + return self.reverse_cw_host_config_of + + def init_mapping(self, mapping): + for key, options in mapping: + if isinstance(key, tuple): # relation definition + assert len(key) == 3 + restrictions = ['X relation_type RT, RT name %(rt)s'] + kwargs = {'rt': key[1]} + if key[0] != '*': + restrictions.append('X from_entity FT, FT name %(ft)s') + kwargs['ft'] = key[0] + if key[2] != '*': + restrictions.append('X to_entity TT, TT name %(tt)s') + kwargs['tt'] = key[2] + rql = 'Any X WHERE %s' % ','.join(restrictions) + schemarset = self._cw.execute(rql, kwargs) + elif key[0].isupper(): # entity type + schemarset = self._cw.execute('CWEType X WHERE X name %(et)s', + {'et': key}) + else: # relation type + schemarset = self._cw.execute('CWRType X WHERE X name %(rt)s', + {'rt': key}) + for schemaentity in schemarset.entities(): + self._cw.create_entity('CWSourceSchemaConfig', + cw_for_source=self, + cw_schema=schemaentity, + options=options) + + +class CWSourceHostConfig(_CWSourceCfgMixIn, AnyEntity): + __regid__ = 'CWSourceHostConfig' + fetch_attrs, fetch_order = fetch_config(['match_host', 'config']) + + def match(self, hostname): + return re.match(self.match_host, hostname) + + +class CWSourceSchemaConfig(AnyEntity): + __regid__ = 'CWSourceSchemaConfig' + fetch_attrs, fetch_order = fetch_config(['cw_for_source', 'cw_schema', 'options']) + + def dc_title(self): + return self._cw._(self.__regid__) + ' #%s' % self.eid + + @property + def schema(self): + return self.cw_schema[0] + + @property + def source(self): + """repository only property, not available from the web side (eg + self._cw is expected to be a server session) + """ + return self._cw.repo.sources_by_eid[self.cw_for_source[0].eid] diff -r 406a41c25e13 -r 0cf10429ad39 hooks/syncsources.py --- a/hooks/syncsources.py Mon Feb 07 15:13:05 2011 +0100 +++ b/hooks/syncsources.py Mon Feb 07 18:19:36 2011 +0100 @@ -37,12 +37,64 @@ raise ValidationError(self.entity.eid, {None: 'cant remove system source'}) SourceRemovedOp(self._cw, uri=self.entity.name) -class SourceRemovedHook(SourceHook): - __regid__ = 'cw.sources.removed' - __select__ = SourceHook.__select__ & hook.match_rtype('cw_support', 'cw_may_cross') - events = ('after_add_relation',) +# source mapping synchronization. Expect cw_for_source/cw_schema are immutable +# relations (i.e. can't change from a source or schema to another). + +class SourceMappingDeleteHook(SourceHook): + """check cw_for_source and cw_schema are immutable relations + + XXX empty delete perms would be enough? + """ + __regid__ = 'cw.sources.delschemaconfig' + __select__ = SourceHook.__select__ & hook.match_rtype('cw_for_source', 'cw_schema') + events = ('before_add_relation',) def __call__(self): - entity = self._cw.entity_from_eid(self.eidto) - if entity.__regid__ == 'CWRType' and entity.name in ('is', 'is_instance_of', 'cw_source'): - msg = self._cw._('the %s relation type can\'t be used here') % entity.name - raise ValidationError(self.eidto, {role_name(self.rtype, 'subject'): msg}) + if not self._cw.added_in_transaction(self.eidfrom): + msg = self._cw._("can't change this relation") + raise ValidationError(self.eidfrom, {self.rtype: msg}) + + +class SourceMappingChangedOp(hook.DataOperationMixIn, hook.Operation): + def check_or_update(self, checkonly): + session = self.session + # take care, can't call get_data() twice + try: + data = self.__data + except AttributeError: + data = self.__data = self.get_data() + for schemacfg, source in data: + if source is None: + source = schemacfg.source + if session.added_in_transaction(schemacfg.eid): + if not session.deleted_in_transaction(schemacfg.eid): + source.add_schema_config(schemacfg, checkonly=checkonly) + elif session.deleted_in_transaction(schemacfg.eid): + source.delete_schema_config(schemacfg, checkonly=checkonly) + else: + source.update_schema_config(schemacfg, checkonly=checkonly) + + def precommit_event(self): + self.check_or_update(True) + + def postcommit_event(self): + self.check_or_update(False) + + +class SourceMappingChangedHook(SourceHook): + __regid__ = 'cw.sources.schemaconfig' + __select__ = SourceHook.__select__ & is_instance('CWSourceSchemaConfig') + events = ('after_add_entity', 'after_update_entity') + def __call__(self): + if self.event == 'after_add_entity' or ( + self.event == 'after_update_entity' and 'options' in self.entity.cw_edited): + SourceMappingChangedOp.get_instance(self._cw).add_data( + (self.entity, None) ) + +class SourceMappingDeleteHook(SourceHook): + __regid__ = 'cw.sources.delschemaconfig' + __select__ = SourceHook.__select__ & hook.match_rtype('cw_for_source') + events = ('before_delete_relation',) + def __call__(self): + SourceMappingChangedOp.get_instance(self._cw).add_data( + (self._cw.entity_from_eid(self.eidfrom), + self._cw.entity_from_eid(self.eidto)) ) diff -r 406a41c25e13 -r 0cf10429ad39 misc/migration/3.11.0_Any.py --- a/misc/migration/3.11.0_Any.py Mon Feb 07 15:13:05 2011 +0100 +++ b/misc/migration/3.11.0_Any.py Mon Feb 07 18:19:36 2011 +0100 @@ -1,6 +1,6 @@ -sync_schema_props_perms('cw_support', syncperms=False) -sync_schema_props_perms('cw_dont_cross', syncperms=False) -sync_schema_props_perms('cw_may_cross', syncperms=False) +for rtype in ('cw_support', 'cw_dont_cross', 'cw_may_cross'): + drop_relation_type(rtype) +add_entity_type('CWSourceSchemaConfig') try: from cubicweb.server.sources.pyrorql import PyroRQLSource @@ -9,7 +9,7 @@ else: from os.path import join - + # function to read old python mapping file def load_mapping_file(source): mappingfile = source.config['mapping-file'] mappingfile = join(source.repo.config.apphome, mappingfile) @@ -37,21 +37,33 @@ assert rtype not in mapping['support_relations'], \ '%s relation should not be in support_relations' % rtype return mapping - + # for now, only pyrorql sources have a mapping for source in repo.sources_by_uri.values(): if not isinstance(source, PyroRQLSource): continue + sourceentity = session.entity_from_eid(source.eid) mapping = load_mapping_file(source) + # write mapping as entities print 'migrating map for', source - for etype in mapping['support_entities']: # XXX write support - rql('SET S cw_support ET WHERE ET name %(etype)s, ET is CWEType, S eid %(s)s', - {'etype': etype, 's': source.eid}) - for rtype in mapping['support_relations']: # XXX write support - rql('SET S cw_support RT WHERE RT name %(rtype)s, RT is CWRType, S eid %(s)s', - {'rtype': rtype, 's': source.eid}) - for rtype in mapping['dont_cross_relations']: # XXX write support - rql('SET S cw_dont_cross RT WHERE RT name %(rtype)s, S eid %(s)s', - {'rtype': rtype, 's': source.eid}) - for rtype in mapping['cross_relations']: # XXX write support - rql('SET S cw_may_cross RT WHERE RT name %(rtype)s, S eid %(s)s', - {'rtype': rtype, 's': source.eid}) + for etype, write in mapping['support_entities'].items(): + create_entity('CWSourceSchemaConfig', + cw_for_source=sourceentity, + cw_schema=session.entity_from_eid(schema[etype].eid), + options=write and u'write' or None, + ask_confirm=False) + for rtype, write in mapping['support_relations'].items(): + options = [] + if write: + options.append(u'write') + if rtype in mapping['cross_relations']: + options.append(u'maycross') + create_entity('CWSourceSchemaConfig', + cw_for_source=sourceentity, + cw_schema=session.entity_from_eid(schema[rtype].eid), + options=u':'.join(options) or None, + ask_confirm=False) + for rtype in mapping['dont_cross_relations']: + create_entity('CWSourceSchemaConfig', + cw_for_source=source, + cw_schema=session.entity_from_eid(schema[etype].eid), + options=u'dontcross') diff -r 406a41c25e13 -r 0cf10429ad39 schema.py --- a/schema.py Mon Feb 07 15:13:05 2011 +0100 +++ b/schema.py Mon Feb 07 18:19:36 2011 +0100 @@ -108,7 +108,7 @@ } PUB_SYSTEM_ATTR_PERMS = { 'read': ('managers', 'users', 'guests',), - 'update': ('managers',), + 'update': ('managers',), } RO_REL_PERMS = { 'read': ('managers', 'users', 'guests',), diff -r 406a41c25e13 -r 0cf10429ad39 schemas/base.py --- a/schemas/base.py Mon Feb 07 15:13:05 2011 +0100 +++ b/schemas/base.py Mon Feb 07 18:19:36 2011 +0100 @@ -307,22 +307,18 @@ cardinality = '1*' composite = 'object' -class cw_support(RelationDefinition): - subject = 'CWSource' - object = ('CWEType', 'CWRType') - constraints = [RQLConstraint('NOT O final TRUE')] +class CWSourceSchemaConfig(EntityType): + __permissions__ = ENTITY_MANAGERS_PERMISSIONS + __unique_together__ = [('cw_for_source', 'cw_schema')] + cw_for_source = SubjectRelation( + 'CWSource', inlined=True, cardinality='1*', composite='object', + __permissions__=RELATION_MANAGERS_PERMISSIONS) + cw_schema = SubjectRelation( + ('CWEType', 'CWRType', 'CWAttribute', 'CWRelation'), + inlined=True, cardinality='1*', composite='object', + __permissions__=RELATION_MANAGERS_PERMISSIONS) + options = String(description=_('allowed options depends on the source type')) -class cw_dont_cross(RelationDefinition): - subject = 'CWSource' - object = 'CWRType' - constraints = [RQLConstraint('NOT O final TRUE'), - RQLConstraint('NOT S cw_may_cross O')] - -class cw_may_cross(RelationDefinition): - subject = 'CWSource' - object = 'CWRType' - constraints = [RQLConstraint('NOT O final TRUE'), - RQLConstraint('NOT S cw_dont_cross O')] # "abtract" relation types, no definition in cubicweb itself ################### diff -r 406a41c25e13 -r 0cf10429ad39 server/sources/__init__.py --- a/server/sources/__init__.py Mon Feb 07 15:13:05 2011 +0100 +++ b/server/sources/__init__.py Mon Feb 07 18:19:36 2011 +0100 @@ -1,4 +1,4 @@ -# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved. +# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved. # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr # # This file is part of CubicWeb. @@ -282,6 +282,21 @@ """ pass + def add_schema_config(self, schemacfg, checkonly=False): + """added CWSourceSchemaConfig, modify mapping accordingly""" + msg = schemacfg._cw._("this source doesn't use a mapping") + raise ValidationError(schemacfg.eid, {None: msg}) + + def del_schema_config(self, schemacfg, checkonly=False): + """deleted CWSourceSchemaConfig, modify mapping accordingly""" + msg = schemacfg._cw._("this source doesn't use a mapping") + raise ValidationError(schemacfg.eid, {None: msg}) + + def update_schema_config(self, schemacfg, checkonly=False): + """updated CWSourceSchemaConfig, modify mapping accordingly""" + self.del_schema_config(schemacfg, checkonly) + self.add_schema_config(schemacfg, checkonly) + # user authentication api ################################################## def authenticate(self, session, login, **kwargs): diff -r 406a41c25e13 -r 0cf10429ad39 server/sources/pyrorql.py --- a/server/sources/pyrorql.py Mon Feb 07 15:13:05 2011 +0100 +++ b/server/sources/pyrorql.py Mon Feb 07 18:19:36 2011 +0100 @@ -1,4 +1,4 @@ -# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved. +# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved. # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr # # This file is part of CubicWeb. @@ -31,11 +31,14 @@ from logilab.common.configuration import REQUIRED from logilab.common.optik_ext import check_yn +from yams.schema import role_name + from rql.nodes import Constant from rql.utils import rqlvar_maker from cubicweb import dbapi, server -from cubicweb import BadConnectionId, UnknownEid, ConnectionError +from cubicweb import ValidationError, BadConnectionId, UnknownEid, ConnectionError +from cubicweb.schema import VIRTUAL_RTYPES from cubicweb.cwconfig import register_persistent_options from cubicweb.server.sources import (AbstractSource, ConnectionWrapper, TimedCache, dbg_st_search, dbg_results) @@ -176,32 +179,87 @@ self.dont_cross_relations = set(('owned_by', 'created_by')) self.cross_relations = set() assert self.eid is not None + self._schemacfg_idx = {} if session is None: _session = self.repo.internal_session() else: _session = session try: - for rql, struct in [('Any ETN WHERE S cw_support ET, ET name ETN, ET is CWEType, S eid %(s)s', - self.support_entities), - ('Any RTN WHERE S cw_support RT, RT name RTN, RT is CWRType, S eid %(s)s', - self.support_relations)]: - for ertype, in _session.execute(rql, {'s': self.eid}): - struct[ertype] = True # XXX write support - for rql, struct in [('Any RTN WHERE S cw_may_cross RT, RT name RTN, S eid %(s)s', - self.cross_relations), - ('Any RTN WHERE S cw_dont_cross RT, RT name RTN, S eid %(s)s', - self.dont_cross_relations)]: - for rtype, in _session.execute(rql, {'s': self.eid}): - struct.add(rtype) + for schemacfg in _session.execute( + 'Any CFG,CFGO,SN,S WHERE ' + 'CFG options CFGO, CFG cw_schema S, S name SN, ' + 'CFG cw_for_source X, X eid %(x)s', {'x': self.eid}).entities(): + self.add_schema_config(schemacfg) finally: if session is None: _session.close() - # XXX move in hooks or schema constraints - for rtype in ('is', 'is_instance_of', 'cw_source'): - assert rtype not in self.dont_cross_relations, \ - '%s relation should not be in dont_cross_relations' % rtype - assert rtype not in self.support_relations, \ - '%s relation should not be in support_relations' % rtype + + etype_options = set(('write',)) + rtype_options = set(('maycross', 'dontcross', 'write',)) + + def _check_options(self, schemacfg, allowedoptions): + if schemacfg.options: + options = set(w.strip() for w in schemacfg.options.split(':')) + else: + options = set() + if options - allowedoptions: + options = ', '.join(sorted(options - allowedoptions)) + msg = _('unknown option(s): %s' % options) + raise ValidationError(schemacfg.eid, {role_name('options', 'subject'): msg}) + return options + + def add_schema_config(self, schemacfg, checkonly=False): + """added CWSourceSchemaConfig, modify mapping accordingly""" + try: + ertype = schemacfg.schema.name + except AttributeError: + msg = schemacfg._cw._("attribute/relation can't be mapped, only " + "entity and relation types") + raise ValidationError(schemacfg.eid, {role_name('cw_for_schema', 'subject'): msg}) + if schemacfg.schema.__regid__ == 'CWEType': + options = self._check_options(schemacfg, self.etype_options) + if not checkonly: + self.support_entities[ertype] = 'write' in options + else: # CWRType + if ertype in ('is', 'is_instance_of', 'cw_source') or ertype in VIRTUAL_RTYPES: + msg = schemacfg._cw._('%s relation should not be in mapped') % rtype + raise ValidationError(schemacfg.eid, {role_name('cw_for_schema', 'subject'): msg}) + options = self._check_options(schemacfg, self.rtype_options) + if 'dontcross' in options: + if 'maycross' in options: + msg = schemacfg._("can't mix dontcross and maycross options") + raise ValidationError(schemacfg.eid, {role_name('options', 'subject'): msg}) + if 'write' in options: + msg = schemacfg._("can't mix dontcross and write options") + raise ValidationError(schemacfg.eid, {role_name('options', 'subject'): msg}) + if not checkonly: + self.dont_cross_relations.add(ertype) + elif not checkonly: + self.support_relations[ertype] = 'write' in options + if 'maycross' in options: + self.cross_relations.add(ertype) + if not checkonly: + # add to an index to ease deletion handling + self._schemacfg_idx[schemacfg.eid] = ertype + + def del_schema_config(self, schemacfg, checkonly=False): + """deleted CWSourceSchemaConfig, modify mapping accordingly""" + if checkonly: + return + try: + ertype = self._schemacfg_idx[schemacfg.eid] + if ertype[0].isupper(): + del self.support_entities[ertype] + else: + if ertype in self.support_relations: + del self.support_relations[ertype] + if ertype in self.cross_relations: + self.cross_relations.remove(ertype) + else: + self.dont_cross_relations.remove(ertype) + except: + self.error('while updating mapping consequently to removal of %s', + schemacfg) def local_eid(self, cnx, extid, session): etype, dexturi, dextid = cnx.describe(extid) diff -r 406a41c25e13 -r 0cf10429ad39 server/test/unittest_msplanner.py --- a/server/test/unittest_msplanner.py Mon Feb 07 15:13:05 2011 +0100 +++ b/server/test/unittest_msplanner.py Mon Feb 07 18:19:36 2011 +0100 @@ -63,7 +63,7 @@ {'X': 'CWConstraint'}, {'X': 'CWConstraintType'}, {'X': 'CWEType'}, {'X': 'CWGroup'}, {'X': 'CWPermission'}, {'X': 'CWProperty'}, {'X': 'CWRType'}, {'X': 'CWRelation'}, - {'X': 'CWSource'}, {'X': 'CWSourceHostConfig'}, + {'X': 'CWSource'}, {'X': 'CWSourceHostConfig'}, {'X': 'CWSourceSchemaConfig'}, {'X': 'CWUser'}, {'X': 'CWUniqueTogetherConstraint'}, {'X': 'Card'}, {'X': 'Comment'}, {'X': 'Division'}, {'X': 'Email'}, {'X': 'EmailAddress'}, {'X': 'EmailPart'}, @@ -897,6 +897,7 @@ ueid = self.session.user.eid ALL_SOLS = X_ALL_SOLS[:] ALL_SOLS.remove({'X': 'CWSourceHostConfig'}) # not authorized + ALL_SOLS.remove({'X': 'CWSourceSchemaConfig'}) # not authorized self._test('Any MAX(X)', [('FetchStep', [('Any E WHERE E type "X", E is Note', [{'E': 'Note'}])], [self.cards, self.system], None, {'E': 'table1.C0'}, []), @@ -947,7 +948,7 @@ ueid = self.session.user.eid X_ET_ALL_SOLS = [] for s in X_ALL_SOLS: - if s == {'X': 'CWSourceHostConfig'}: + if s in ({'X': 'CWSourceHostConfig'}, {'X': 'CWSourceSchemaConfig'}): continue # not authorized ets = {'ET': 'CWEType'} ets.update(s) @@ -2539,7 +2540,7 @@ None, {'X': 'table0.C0'}, []), ('UnionStep', None, None, [('OneFetchStep', - [(u'Any X WHERE X owned_by U, U login "anon", U is CWUser, X is IN(Affaire, BaseTransition, Basket, Bookmark, CWAttribute, CWCache, CWConstraint, CWConstraintType, CWEType, CWGroup, CWPermission, CWProperty, CWRType, CWRelation, CWSource, CWSourceHostConfig, CWUniqueTogetherConstraint, CWUser, Division, Email, EmailAddress, EmailPart, EmailThread, ExternalUri, File, Folder, Personne, RQLExpression, Societe, SubDivision, SubWorkflowExitPoint, Tag, TrInfo, Transition, Workflow, WorkflowTransition)', + [(u'Any X WHERE X owned_by U, U login "anon", U is CWUser, X is IN(Affaire, BaseTransition, Basket, Bookmark, CWAttribute, CWCache, CWConstraint, CWConstraintType, CWEType, CWGroup, CWPermission, CWProperty, CWRType, CWRelation, CWSource, CWSourceHostConfig, CWSourceSchemaConfig, CWUniqueTogetherConstraint, CWUser, Division, Email, EmailAddress, EmailPart, EmailThread, ExternalUri, File, Folder, Personne, RQLExpression, Societe, SubDivision, SubWorkflowExitPoint, Tag, TrInfo, Transition, Workflow, WorkflowTransition)', [{'U': 'CWUser', 'X': 'Affaire'}, {'U': 'CWUser', 'X': 'BaseTransition'}, {'U': 'CWUser', 'X': 'Basket'}, @@ -2556,6 +2557,7 @@ {'U': 'CWUser', 'X': 'CWRelation'}, {'U': 'CWUser', 'X': 'CWSource'}, {'U': 'CWUser', 'X': 'CWSourceHostConfig'}, + {'U': 'CWUser', 'X': 'CWSourceSchemaConfig'}, {'U': 'CWUser', 'X': 'CWUniqueTogetherConstraint'}, {'U': 'CWUser', 'X': 'CWUser'}, {'U': 'CWUser', 'X': 'Division'}, diff -r 406a41c25e13 -r 0cf10429ad39 server/test/unittest_multisources.py --- a/server/test/unittest_multisources.py Mon Feb 07 15:13:05 2011 +0100 +++ b/server/test/unittest_multisources.py Mon Feb 07 18:19:36 2011 +0100 @@ -1,4 +1,4 @@ -# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved. +# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved. # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr # # This file is part of CubicWeb. @@ -17,6 +17,7 @@ # with CubicWeb. If not, see . from datetime import datetime, timedelta +from itertools import repeat from cubicweb.devtools import TestServerConfiguration, init_test_database from cubicweb.devtools.testlib import CubicWebTC, refresh_repo @@ -46,13 +47,9 @@ Connection_close = Connection.close def add_extern_mapping(source): - execute = source._cw.execute - for etype in ('Card', 'Affaire', 'State'): - assert execute('SET S cw_support ET WHERE ET name %(etype)s, ET is CWEType, S eid %(s)s', - {'etype': etype, 's': source.eid}) - for rtype in ('in_state', 'documented_by', 'multisource_inlined_rel'): - assert execute('SET S cw_support RT WHERE RT name %(rtype)s, RT is CWRType, S eid %(s)s', - {'rtype': rtype, 's': source.eid}) + source.init_mapping(zip(('Card', 'Affaire', 'State', + 'in_state', 'documented_by', 'multisource_inlined_rel'), + repeat(u'write'))) def setUpModule(*args): @@ -63,6 +60,7 @@ repo3, cnx3 = init_test_database(config=cfg2) src = cnx3.request().create_entity('CWSource', name=u'extern', type=u'pyrorql', config=EXTERN_SOURCE_CFG) + cnx3.commit() # must commit before adding the mapping add_extern_mapping(src) cnx3.commit() @@ -122,6 +120,7 @@ source = self.request().create_entity( 'CWSource', name=unicode(uri), type=u'pyrorql', config=unicode(config)) + self.commit() # must commit before adding the mapping add_extern_mapping(source) self.commit() # trigger discovery diff -r 406a41c25e13 -r 0cf10429ad39 server/test/unittest_querier.py --- a/server/test/unittest_querier.py Mon Feb 07 15:13:05 2011 +0100 +++ b/server/test/unittest_querier.py Mon Feb 07 18:19:36 2011 +0100 @@ -501,15 +501,15 @@ [[u'description_format', 12], [u'description', 13], [u'name', 15], - [u'created_by', 40], - [u'creation_date', 40], - [u'cw_source', 40], - [u'cwuri', 40], - [u'in_basket', 40], - [u'is', 40], - [u'is_instance_of', 40], - [u'modification_date', 40], - [u'owned_by', 40]]) + [u'created_by', 41], + [u'creation_date', 41], + [u'cw_source', 41], + [u'cwuri', 41], + [u'in_basket', 41], + [u'is', 41], + [u'is_instance_of', 41], + [u'modification_date', 41], + [u'owned_by', 41]]) def test_select_aggregat_having_dumb(self): # dumb but should not raise an error diff -r 406a41c25e13 -r 0cf10429ad39 test/unittest_schema.py --- a/test/unittest_schema.py Mon Feb 07 15:13:05 2011 +0100 +++ b/test/unittest_schema.py Mon Feb 07 18:19:36 2011 +0100 @@ -163,7 +163,7 @@ 'CWCache', 'CWConstraint', 'CWConstraintType', 'CWEType', 'CWAttribute', 'CWGroup', 'EmailAddress', 'CWRelation', 'CWPermission', 'CWProperty', 'CWRType', - 'CWSource', 'CWSourceHostConfig', + 'CWSource', 'CWSourceHostConfig', 'CWSourceSchemaConfig', 'CWUniqueTogetherConstraint', 'CWUser', 'ExternalUri', 'File', 'Float', 'Int', 'Interval', 'Note', 'Password', 'Personne', @@ -171,7 +171,7 @@ 'Societe', 'State', 'StateFull', 'String', 'SubNote', 'SubWorkflowExitPoint', 'Tag', 'Time', 'Transition', 'TrInfo', 'Workflow', 'WorkflowTransition'] - self.assertListEqual(entities, sorted(expected_entities)) + self.assertListEqual(sorted(expected_entities), entities) relations = sorted([str(r) for r in schema.relations()]) expected_relations = ['add_permission', 'address', 'alias', 'allowed_transition', 'bookmarked_by', 'by_transition', @@ -181,8 +181,7 @@ 'constrained_by', 'constraint_of', 'content', 'content_format', 'created_by', 'creation_date', 'cstrtype', 'custom_workflow', - 'cwuri', 'cw_source', 'cw_host_config_of', - 'cw_support', 'cw_dont_cross', 'cw_may_cross', + 'cwuri', 'cw_for_source', 'cw_host_config_of', 'cw_schema', 'cw_source', 'data', 'data_encoding', 'data_format', 'data_name', 'default_workflow', 'defaultval', 'delete_permission', 'description', 'description_format', 'destination_state', @@ -202,7 +201,7 @@ 'name', 'nom', - 'ordernum', 'owned_by', + 'options', 'ordernum', 'owned_by', 'path', 'pkey', 'prefered_form', 'prenom', 'primary_email', @@ -218,7 +217,7 @@ 'wf_info_for', 'wikiid', 'workflow_of', 'tr_count'] - self.assertListEqual(relations, sorted(expected_relations)) + self.assertListEqual(sorted(expected_relations), relations) eschema = schema.eschema('CWUser') rels = sorted(str(r) for r in eschema.subject_relations()) diff -r 406a41c25e13 -r 0cf10429ad39 web/views/cwsources.py --- a/web/views/cwsources.py Mon Feb 07 15:13:05 2011 +0100 +++ b/web/views/cwsources.py Mon Feb 07 18:19:36 2011 +0100 @@ -1,4 +1,4 @@ -# copyright 2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved. +# copyright 2010-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved. # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr # # This file is part of CubicWeb. @@ -15,22 +15,24 @@ # # You should have received a copy of the GNU Lesser General Public License along # with CubicWeb. If not, see . -"""Specific views for data sources""" +"""Specific views for data sources and related entities (eg CWSource, +CWSourceHostConfig, CWSourceSchemaConfig). +""" __docformat__ = "restructuredtext en" _ = unicode from itertools import repeat, chain -from cubicweb.selectors import is_instance, score_entity +from cubicweb.selectors import is_instance, score_entity, match_user_groups from cubicweb.view import EntityView from cubicweb.schema import META_RTYPES, VIRTUAL_RTYPES, display_name from cubicweb.web import uicfg from cubicweb.web.views import tabs -for rtype in ('cw_support', 'cw_may_cross', 'cw_dont_cross'): - uicfg.primaryview_section.tag_subject_of(('CWSource', rtype, '*'), - 'hidden') +_pvs = uicfg.primaryview_section +_pvs.tag_object_of(('*', 'cw_for_source', 'CWSource'), 'hidden') + class CWSourcePrimaryView(tabs.TabbedPrimaryView): __select__ = is_instance('CWSource') @@ -43,129 +45,155 @@ __select__ = tabs.PrimaryTab.__select__ & is_instance('CWSource') +MAPPED_SOURCE_TYPES = set( ('pyrorql', 'datafeed') ) + class CWSourceMappingTab(EntityView): __regid__ = 'cwsource-mapping' __select__ = (tabs.PrimaryTab.__select__ & is_instance('CWSource') - & score_entity(lambda x:x.type == 'pyrorql')) + & match_user_groups('managers') + & score_entity(lambda x:x.type in MAPPED_SOURCE_TYPES)) def entity_call(self, entity): _ = self._cw._ - self.w('

%s

' % _('Entity and relation types supported by this source')) - self.wview('list', entity.related('cw_support'), 'noresult') - self.w('

%s

' % _('Relations that should not be crossed')) - self.w('

%s

' % _( - 'By default, when a relation is not supported by a source, it is ' - 'supposed that a local relation may point to an entity from the ' - 'external source. Relations listed here won\'t have this ' - '"crossing" behaviour.')) - self.wview('list', entity.related('cw_dont_cross'), 'noresult') - self.w('

%s

' % _('Relations that can be crossed')) - self.w('

%s

' % _( - 'By default, when a relation is supported by a source, it is ' - 'supposed that a local relation can\'t point to an entity from the ' - 'external source. Relations listed here may have this ' - '"crossing" behaviour anyway.')) - self.wview('list', entity.related('cw_may_cross'), 'noresult') - if self._cw.user.is_in_group('managers'): - errors, warnings, infos = check_mapping(entity) - if (errors or warnings or infos): + self.w('

%s

' % _('Entity and relation supported by this source')) + rset = self._cw.execute( + 'Any X, SCH, XO ORDERBY ET WHERE X options XO, X cw_for_source S, S eid %(s)s, ' + 'X cw_schema SCH, SCH is ET', {'s': entity.eid}) + self.wview('table', rset, 'noresult') + # self.w('

%s

' % _('Relations that should not be crossed')) + # self.w('

%s

' % _( + # 'By default, when a relation is not supported by a source, it is ' + # 'supposed that a local relation may point to an entity from the ' + # 'external source. Relations listed here won\'t have this ' + # '"crossing" behaviour.')) + # self.wview('list', entity.related('cw_dont_cross'), 'noresult') + # self.w('

%s

' % _('Relations that can be crossed')) + # self.w('

%s

' % _( + # 'By default, when a relation is supported by a source, it is ' + # 'supposed that a local relation can\'t point to an entity from the ' + # 'external source. Relations listed here may have this ' + # '"crossing" behaviour anyway.')) + # self.wview('list', entity.related('cw_may_cross'), 'noresult') + checker = MAPPING_CHECKERS.get(entity.type, MappingChecker)(entity) + checker.check() + if (checker.errors or checker.warnings or checker.infos): self.w('

%s

' % _('Detected problems')) - errors = zip(repeat(_('error'), errors)) - warnings = zip(repeat(_('warning'), warnings)) - infos = zip(repeat(_('warning'), infos)) + errors = zip(repeat(_('error')), checker.errors) + warnings = zip(repeat(_('warning')), checker.warnings) + infos = zip(repeat(_('warning')), checker.infos) self.wview('pyvaltable', pyvalue=chain(errors, warnings, infos)) -def check_mapping(cwsource): - req = cwsource._cw - _ = req._ - errors = [] - error = errors.append - warnings = [] - warning = warnings.append - infos = [] - info = infos.append - srelations = set() - sentities = set() - maycross = set() - dontcross = set() - # first check supported stuff / meta & virtual types and get mapping as sets - for cwertype in cwsource.cw_support: - if cwertype.name in META_RTYPES: - error(_('meta relation %s can not be supported') % cwertype.name) - else: - if cwertype.__regid__ == 'CWEType': - sentities.add(cwertype.name) - else: - srelations.add(cwertype.name) - for attr, attrset in (('cw_may_cross', maycross), - ('cw_dont_cross', dontcross)): - for cwrtype in getattr(cwsource, attr): - if cwrtype.name in VIRTUAL_RTYPES: - error(_('virtual relation %(rtype)s can not be referenced by ' - 'the "%(srel)s" relation') % - {'rtype': cwrtype.name, - 'srel': display_name(req, attr, context='CWSource')}) + +class MappingChecker(object): + def __init__(self, cwsource): + self.cwsource = cwsource + self.errors = [] + self.warnings = [] + self.infos = [] + self.schema = cwsource._cw.vreg.schema + + def init(self): + # supported entity types + self.sentities = set() + # supported relations + self.srelations = {} + # avoid duplicated messages + self.seen = set() + # first get mapping as dict/sets + for schemacfg in self.cwsource.reverse_cw_for_source: + self.init_schemacfg(schemacfg) + + def init_schemacfg(self, schemacfg): + cwerschema = schemacfg.schema + if cwerschema.__regid__ == 'CWEType': + self.sentities.add(cwerschema.name) + elif cwerschema.__regid__ == 'CWRType': + assert not cwerschema.name in self.srelations + self.srelations[cwerschema.name] = None + else: # CWAttribute/CWRelation + self.srelations.setdefault(cwerschema.rtype.name, []).append( + (cwerschema.stype.name, cwerschema.otype.name) ) + + def check(self): + self.init() + error = self.errors.append + warning = self.warnings.append + info = self.infos.append + for etype in self.sentities: + eschema = self.schema[etype] + for rschema, ttypes, role in eschema.relation_definitions(): + if rschema in META_RTYPES: + continue + ttypes = [ttype for ttype in ttypes if ttype in self.sentities] + if not rschema in self.srelations: + for ttype in ttypes: + rdef = rschema.role_rdef(etype, ttype, role) + self.seen.add(rdef) + if rdef.role_cardinality(role) in '1+': + error(_('relation %(type)s with %(etype)s as %(role)s ' + 'and target type %(target)s is mandatory but ' + 'not supported') % + {'rtype': rschema, 'etype': etype, 'role': role, + 'target': ttype}) + elif ttype in self.sentities: + warning(_('%s could be supported') % rdef) + elif not ttypes: + warning(_('relation %(rtype)s with %(etype)s as %(role)s is ' + 'supported but no target type supported') % + {'rtype': rschema, 'role': role, 'etype': etype}) + for rtype in self.srelations: + rschema = self.schema[rtype] + for subj, obj in rschema.rdefs: + if subj in self.sentities and obj in self.sentities: + break else: - attrset.add(cwrtype.name) - # check relation in dont_cross_relations aren't in support_relations - for rtype in dontcross & maycross: - info(_('relation %(rtype)s is supported but in %(dontcross)s') % - {'rtype': rtype, - 'dontcross': display_name(req, 'cw_dont_cross', - context='CWSource')}) - # check relation in cross_relations are in support_relations - for rtype in maycross & srelations: - info(_('relation %(rtype)s isn\'t supported but in %(maycross)s') % - {'rtype': rtype, - 'dontcross': display_name(req, 'cw_may_cross', - context='CWSource')}) - # now check for more handy things - seen = set() - for etype in sentities: - eschema = req.vreg.schema[etype] - for rschema, ttypes, role in eschema.relation_definitions(): - if rschema in META_RTYPES: - continue - ttypes = [ttype for ttype in ttypes if ttype in sentities] - if not rschema in srelations: - somethingprinted = False - for ttype in ttypes: - rdef = rschema.role_rdef(etype, ttype, role) - seen.add(rdef) - if rdef.role_cardinality(role) in '1+': - error(_('relation %(type)s with %(etype)s as %(role)s ' - 'and target type %(target)s is mandatory but ' - 'not supported') % - {'rtype': rschema, 'etype': etype, 'role': role, - 'target': ttype}) - somethingprinted = True - elif ttype in sentities: - if rdef not in seen: - warning(_('%s could be supported') % rdef) - somethingprinted = True - if rschema not in dontcross: - if role == 'subject' and rschema.inlined: - error(_('inlined relation %(rtype)s of %(etype)s ' - 'should be supported') % - {'rtype': rschema, 'etype': etype}) - elif (not somethingprinted and rschema not in seen - and rschema not in maycross): - info(_('you may want to specify something for %s') % - rschema) - seen.add(rschema) - else: - if not ttypes: - warning(_('relation %(rtype)s with %(etype)s as %(role)s ' - 'is supported but no target type supported') % - {'rtype': rschema, 'role': role, 'etype': etype}) - if rschema in maycross and rschema.inlined: + error(_('relation %s is supported but none if its definitions ' + 'matches supported entities') % rtype) + self.custom_check() + + def custom_check(self): + pass + + +class PyroRQLMappingChecker(MappingChecker): + """pyrorql source mapping checker""" + + def init(self): + self.dontcross = set() + self.maycross = set() + super(PyroRQLMappingChecker, self).init() + + def init_schemacfg(self, schemacfg): + options = schemacfg.options or () + if 'dontcross' in options: + self.dontcross.add(schemacfg.schema.name) + else: + super(PyroRQLMappingChecker, self).init_schemacfg(schemacfg) + if 'maycross' in options: + self.maycross.add(schemacfg.schema.name) + + def custom_check(self): + error = self.errors.append + info = self.infos.append + for etype in self.sentities: + eschema = self.schema[etype] + for rschema, ttypes, role in eschema.relation_definitions(): + if rschema in META_RTYPES: + continue + if not rschema in self.srelations: + if rschema not in self.dontcross: + if role == 'subject' and rschema.inlined: + error(_('inlined relation %(rtype)s of %(etype)s ' + 'should be supported') % + {'rtype': rschema, 'etype': etype}) + elif (rschema not in self.seen and rschema not in self.maycross): + info(_('you may want to specify something for %s') % + rschema) + self.seen.add(rschema) + elif rschema in self.maycross and rschema.inlined: error(_('you should un-inline relation %s which is ' 'supported and may be crossed ') % rschema) - for rschema in srelations: - for subj, obj in rschema.rdefs: - if subj in sentities and obj in sentities: - break - else: - error(_('relation %s is supported but none if its definitions ' - 'matches supported entities') % rschema) - return errors, warnings, infos + +MAPPING_CHECKERS = { + 'pyrorql': PyroRQLMappingChecker, + }