[sources] rewrite the way pyrorql mapping are stored in the database so it can be reused for other sources (eg datafeed+cwxml)
--- a/entities/schemaobjs.py Mon Feb 07 15:13:05 2011 +0100
+++ b/entities/schemaobjs.py Mon Feb 07 18:19:36 2011 +0100
@@ -1,4 +1,4 @@
-# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
@@ -19,12 +19,7 @@
__docformat__ = "restructuredtext en"
-import re
-from socket import gethostname
-
from logilab.common.decorators import cached
-from logilab.common.textutils import text_to_dict
-from logilab.common.configuration import OptionError
from yams.schema import role_name
@@ -34,58 +29,6 @@
from cubicweb.entities import AnyEntity, fetch_config
-class _CWSourceCfgMixIn(object):
- @property
- def dictconfig(self):
- return self.config and text_to_dict(self.config) or {}
-
- def update_config(self, skip_unknown=False, **config):
- from cubicweb.server import SOURCE_TYPES
- from cubicweb.server.serverconfig import (SourceConfiguration,
- generate_source_config)
- cfg = self.dictconfig
- cfg.update(config)
- options = SOURCE_TYPES[self.type].options
- sconfig = SourceConfiguration(self._cw.vreg.config, options=options)
- for opt, val in cfg.iteritems():
- try:
- sconfig.set_option(opt, val)
- except OptionError:
- if skip_unknown:
- continue
- raise
- cfgstr = unicode(generate_source_config(sconfig), self._cw.encoding)
- self.set_attributes(config=cfgstr)
-
-
-class CWSource(_CWSourceCfgMixIn, AnyEntity):
- __regid__ = 'CWSource'
- fetch_attrs, fetch_order = fetch_config(['name', 'type'])
-
- @property
- def host_config(self):
- dictconfig = self.dictconfig
- host = gethostname()
- for hostcfg in self.host_configs:
- if hostcfg.match(host):
- self.info('matching host config %s for source %s',
- hostcfg.match_host, self.name)
- dictconfig.update(hostcfg.dictconfig)
- return dictconfig
-
- @property
- def host_configs(self):
- return self.reverse_cw_host_config_of
-
-
-class CWSourceHostConfig(_CWSourceCfgMixIn, AnyEntity):
- __regid__ = 'CWSourceHostConfig'
- fetch_attrs, fetch_order = fetch_config(['match_host', 'config'])
-
- def match(self, hostname):
- return re.match(self.match_host, hostname)
-
-
class CWEType(AnyEntity):
__regid__ = 'CWEType'
fetch_attrs, fetch_order = fetch_config(['name'])
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/entities/sources.py Mon Feb 07 18:19:36 2011 +0100
@@ -0,0 +1,125 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of CubicWeb.
+#
+# CubicWeb is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option)
+# any later version.
+#
+# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
+"""data source related entities"""
+
+__docformat__ = "restructuredtext en"
+
+import re
+from socket import gethostname
+
+from logilab.common.textutils import text_to_dict
+from logilab.common.configuration import OptionError
+
+from cubicweb import ValidationError
+from cubicweb.entities import AnyEntity, fetch_config
+
+class _CWSourceCfgMixIn(object):
+ @property
+ def dictconfig(self):
+ return self.config and text_to_dict(self.config) or {}
+
+ def update_config(self, skip_unknown=False, **config):
+ from cubicweb.server import SOURCE_TYPES
+ from cubicweb.server.serverconfig import (SourceConfiguration,
+ generate_source_config)
+ cfg = self.dictconfig
+ cfg.update(config)
+ options = SOURCE_TYPES[self.type].options
+ sconfig = SourceConfiguration(self._cw.vreg.config, options=options)
+ for opt, val in cfg.iteritems():
+ try:
+ sconfig.set_option(opt, val)
+ except OptionError:
+ if skip_unknown:
+ continue
+ raise
+ cfgstr = unicode(generate_source_config(sconfig), self._cw.encoding)
+ self.set_attributes(config=cfgstr)
+
+
+class CWSource(_CWSourceCfgMixIn, AnyEntity):
+ __regid__ = 'CWSource'
+ fetch_attrs, fetch_order = fetch_config(['name', 'type'])
+
+ @property
+ def host_config(self):
+ dictconfig = self.dictconfig
+ host = gethostname()
+ for hostcfg in self.host_configs:
+ if hostcfg.match(host):
+ self.info('matching host config %s for source %s',
+ hostcfg.match_host, self.name)
+ dictconfig.update(hostcfg.dictconfig)
+ return dictconfig
+
+ @property
+ def host_configs(self):
+ return self.reverse_cw_host_config_of
+
+ def init_mapping(self, mapping):
+ for key, options in mapping:
+ if isinstance(key, tuple): # relation definition
+ assert len(key) == 3
+ restrictions = ['X relation_type RT, RT name %(rt)s']
+ kwargs = {'rt': key[1]}
+ if key[0] != '*':
+ restrictions.append('X from_entity FT, FT name %(ft)s')
+ kwargs['ft'] = key[0]
+ if key[2] != '*':
+ restrictions.append('X to_entity TT, TT name %(tt)s')
+ kwargs['tt'] = key[2]
+ rql = 'Any X WHERE %s' % ','.join(restrictions)
+ schemarset = self._cw.execute(rql, kwargs)
+ elif key[0].isupper(): # entity type
+ schemarset = self._cw.execute('CWEType X WHERE X name %(et)s',
+ {'et': key})
+ else: # relation type
+ schemarset = self._cw.execute('CWRType X WHERE X name %(rt)s',
+ {'rt': key})
+ for schemaentity in schemarset.entities():
+ self._cw.create_entity('CWSourceSchemaConfig',
+ cw_for_source=self,
+ cw_schema=schemaentity,
+ options=options)
+
+
+class CWSourceHostConfig(_CWSourceCfgMixIn, AnyEntity):
+ __regid__ = 'CWSourceHostConfig'
+ fetch_attrs, fetch_order = fetch_config(['match_host', 'config'])
+
+ def match(self, hostname):
+ return re.match(self.match_host, hostname)
+
+
+class CWSourceSchemaConfig(AnyEntity):
+ __regid__ = 'CWSourceSchemaConfig'
+ fetch_attrs, fetch_order = fetch_config(['cw_for_source', 'cw_schema', 'options'])
+
+ def dc_title(self):
+ return self._cw._(self.__regid__) + ' #%s' % self.eid
+
+ @property
+ def schema(self):
+ return self.cw_schema[0]
+
+ @property
+ def source(self):
+ """repository only property, not available from the web side (eg
+ self._cw is expected to be a server session)
+ """
+ return self._cw.repo.sources_by_eid[self.cw_for_source[0].eid]
--- a/hooks/syncsources.py Mon Feb 07 15:13:05 2011 +0100
+++ b/hooks/syncsources.py Mon Feb 07 18:19:36 2011 +0100
@@ -37,12 +37,64 @@
raise ValidationError(self.entity.eid, {None: 'cant remove system source'})
SourceRemovedOp(self._cw, uri=self.entity.name)
-class SourceRemovedHook(SourceHook):
- __regid__ = 'cw.sources.removed'
- __select__ = SourceHook.__select__ & hook.match_rtype('cw_support', 'cw_may_cross')
- events = ('after_add_relation',)
+# source mapping synchronization. Expect cw_for_source/cw_schema are immutable
+# relations (i.e. can't change from a source or schema to another).
+
+class SourceMappingDeleteHook(SourceHook):
+ """check cw_for_source and cw_schema are immutable relations
+
+ XXX empty delete perms would be enough?
+ """
+ __regid__ = 'cw.sources.delschemaconfig'
+ __select__ = SourceHook.__select__ & hook.match_rtype('cw_for_source', 'cw_schema')
+ events = ('before_add_relation',)
def __call__(self):
- entity = self._cw.entity_from_eid(self.eidto)
- if entity.__regid__ == 'CWRType' and entity.name in ('is', 'is_instance_of', 'cw_source'):
- msg = self._cw._('the %s relation type can\'t be used here') % entity.name
- raise ValidationError(self.eidto, {role_name(self.rtype, 'subject'): msg})
+ if not self._cw.added_in_transaction(self.eidfrom):
+ msg = self._cw._("can't change this relation")
+ raise ValidationError(self.eidfrom, {self.rtype: msg})
+
+
+class SourceMappingChangedOp(hook.DataOperationMixIn, hook.Operation):
+ def check_or_update(self, checkonly):
+ session = self.session
+ # take care, can't call get_data() twice
+ try:
+ data = self.__data
+ except AttributeError:
+ data = self.__data = self.get_data()
+ for schemacfg, source in data:
+ if source is None:
+ source = schemacfg.source
+ if session.added_in_transaction(schemacfg.eid):
+ if not session.deleted_in_transaction(schemacfg.eid):
+ source.add_schema_config(schemacfg, checkonly=checkonly)
+ elif session.deleted_in_transaction(schemacfg.eid):
+ source.delete_schema_config(schemacfg, checkonly=checkonly)
+ else:
+ source.update_schema_config(schemacfg, checkonly=checkonly)
+
+ def precommit_event(self):
+ self.check_or_update(True)
+
+ def postcommit_event(self):
+ self.check_or_update(False)
+
+
+class SourceMappingChangedHook(SourceHook):
+ __regid__ = 'cw.sources.schemaconfig'
+ __select__ = SourceHook.__select__ & is_instance('CWSourceSchemaConfig')
+ events = ('after_add_entity', 'after_update_entity')
+ def __call__(self):
+ if self.event == 'after_add_entity' or (
+ self.event == 'after_update_entity' and 'options' in self.entity.cw_edited):
+ SourceMappingChangedOp.get_instance(self._cw).add_data(
+ (self.entity, None) )
+
+class SourceMappingDeleteHook(SourceHook):
+ __regid__ = 'cw.sources.delschemaconfig'
+ __select__ = SourceHook.__select__ & hook.match_rtype('cw_for_source')
+ events = ('before_delete_relation',)
+ def __call__(self):
+ SourceMappingChangedOp.get_instance(self._cw).add_data(
+ (self._cw.entity_from_eid(self.eidfrom),
+ self._cw.entity_from_eid(self.eidto)) )
--- a/misc/migration/3.11.0_Any.py Mon Feb 07 15:13:05 2011 +0100
+++ b/misc/migration/3.11.0_Any.py Mon Feb 07 18:19:36 2011 +0100
@@ -1,6 +1,6 @@
-sync_schema_props_perms('cw_support', syncperms=False)
-sync_schema_props_perms('cw_dont_cross', syncperms=False)
-sync_schema_props_perms('cw_may_cross', syncperms=False)
+for rtype in ('cw_support', 'cw_dont_cross', 'cw_may_cross'):
+ drop_relation_type(rtype)
+add_entity_type('CWSourceSchemaConfig')
try:
from cubicweb.server.sources.pyrorql import PyroRQLSource
@@ -9,7 +9,7 @@
else:
from os.path import join
-
+ # function to read old python mapping file
def load_mapping_file(source):
mappingfile = source.config['mapping-file']
mappingfile = join(source.repo.config.apphome, mappingfile)
@@ -37,21 +37,33 @@
assert rtype not in mapping['support_relations'], \
'%s relation should not be in support_relations' % rtype
return mapping
-
+ # for now, only pyrorql sources have a mapping
for source in repo.sources_by_uri.values():
if not isinstance(source, PyroRQLSource):
continue
+ sourceentity = session.entity_from_eid(source.eid)
mapping = load_mapping_file(source)
+ # write mapping as entities
print 'migrating map for', source
- for etype in mapping['support_entities']: # XXX write support
- rql('SET S cw_support ET WHERE ET name %(etype)s, ET is CWEType, S eid %(s)s',
- {'etype': etype, 's': source.eid})
- for rtype in mapping['support_relations']: # XXX write support
- rql('SET S cw_support RT WHERE RT name %(rtype)s, RT is CWRType, S eid %(s)s',
- {'rtype': rtype, 's': source.eid})
- for rtype in mapping['dont_cross_relations']: # XXX write support
- rql('SET S cw_dont_cross RT WHERE RT name %(rtype)s, S eid %(s)s',
- {'rtype': rtype, 's': source.eid})
- for rtype in mapping['cross_relations']: # XXX write support
- rql('SET S cw_may_cross RT WHERE RT name %(rtype)s, S eid %(s)s',
- {'rtype': rtype, 's': source.eid})
+ for etype, write in mapping['support_entities'].items():
+ create_entity('CWSourceSchemaConfig',
+ cw_for_source=sourceentity,
+ cw_schema=session.entity_from_eid(schema[etype].eid),
+ options=write and u'write' or None,
+ ask_confirm=False)
+ for rtype, write in mapping['support_relations'].items():
+ options = []
+ if write:
+ options.append(u'write')
+ if rtype in mapping['cross_relations']:
+ options.append(u'maycross')
+ create_entity('CWSourceSchemaConfig',
+ cw_for_source=sourceentity,
+ cw_schema=session.entity_from_eid(schema[rtype].eid),
+ options=u':'.join(options) or None,
+ ask_confirm=False)
+ for rtype in mapping['dont_cross_relations']:
+ create_entity('CWSourceSchemaConfig',
+ cw_for_source=source,
+ cw_schema=session.entity_from_eid(schema[etype].eid),
+ options=u'dontcross')
--- a/schema.py Mon Feb 07 15:13:05 2011 +0100
+++ b/schema.py Mon Feb 07 18:19:36 2011 +0100
@@ -108,7 +108,7 @@
}
PUB_SYSTEM_ATTR_PERMS = {
'read': ('managers', 'users', 'guests',),
- 'update': ('managers',),
+ 'update': ('managers',),
}
RO_REL_PERMS = {
'read': ('managers', 'users', 'guests',),
--- a/schemas/base.py Mon Feb 07 15:13:05 2011 +0100
+++ b/schemas/base.py Mon Feb 07 18:19:36 2011 +0100
@@ -307,22 +307,18 @@
cardinality = '1*'
composite = 'object'
-class cw_support(RelationDefinition):
- subject = 'CWSource'
- object = ('CWEType', 'CWRType')
- constraints = [RQLConstraint('NOT O final TRUE')]
+class CWSourceSchemaConfig(EntityType):
+ __permissions__ = ENTITY_MANAGERS_PERMISSIONS
+ __unique_together__ = [('cw_for_source', 'cw_schema')]
+ cw_for_source = SubjectRelation(
+ 'CWSource', inlined=True, cardinality='1*', composite='object',
+ __permissions__=RELATION_MANAGERS_PERMISSIONS)
+ cw_schema = SubjectRelation(
+ ('CWEType', 'CWRType', 'CWAttribute', 'CWRelation'),
+ inlined=True, cardinality='1*', composite='object',
+ __permissions__=RELATION_MANAGERS_PERMISSIONS)
+ options = String(description=_('allowed options depends on the source type'))
-class cw_dont_cross(RelationDefinition):
- subject = 'CWSource'
- object = 'CWRType'
- constraints = [RQLConstraint('NOT O final TRUE'),
- RQLConstraint('NOT S cw_may_cross O')]
-
-class cw_may_cross(RelationDefinition):
- subject = 'CWSource'
- object = 'CWRType'
- constraints = [RQLConstraint('NOT O final TRUE'),
- RQLConstraint('NOT S cw_dont_cross O')]
# "abtract" relation types, no definition in cubicweb itself ###################
--- a/server/sources/__init__.py Mon Feb 07 15:13:05 2011 +0100
+++ b/server/sources/__init__.py Mon Feb 07 18:19:36 2011 +0100
@@ -1,4 +1,4 @@
-# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
@@ -282,6 +282,21 @@
"""
pass
+ def add_schema_config(self, schemacfg, checkonly=False):
+ """added CWSourceSchemaConfig, modify mapping accordingly"""
+ msg = schemacfg._cw._("this source doesn't use a mapping")
+ raise ValidationError(schemacfg.eid, {None: msg})
+
+ def del_schema_config(self, schemacfg, checkonly=False):
+ """deleted CWSourceSchemaConfig, modify mapping accordingly"""
+ msg = schemacfg._cw._("this source doesn't use a mapping")
+ raise ValidationError(schemacfg.eid, {None: msg})
+
+ def update_schema_config(self, schemacfg, checkonly=False):
+ """updated CWSourceSchemaConfig, modify mapping accordingly"""
+ self.del_schema_config(schemacfg, checkonly)
+ self.add_schema_config(schemacfg, checkonly)
+
# user authentication api ##################################################
def authenticate(self, session, login, **kwargs):
--- a/server/sources/pyrorql.py Mon Feb 07 15:13:05 2011 +0100
+++ b/server/sources/pyrorql.py Mon Feb 07 18:19:36 2011 +0100
@@ -1,4 +1,4 @@
-# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
@@ -31,11 +31,14 @@
from logilab.common.configuration import REQUIRED
from logilab.common.optik_ext import check_yn
+from yams.schema import role_name
+
from rql.nodes import Constant
from rql.utils import rqlvar_maker
from cubicweb import dbapi, server
-from cubicweb import BadConnectionId, UnknownEid, ConnectionError
+from cubicweb import ValidationError, BadConnectionId, UnknownEid, ConnectionError
+from cubicweb.schema import VIRTUAL_RTYPES
from cubicweb.cwconfig import register_persistent_options
from cubicweb.server.sources import (AbstractSource, ConnectionWrapper,
TimedCache, dbg_st_search, dbg_results)
@@ -176,32 +179,87 @@
self.dont_cross_relations = set(('owned_by', 'created_by'))
self.cross_relations = set()
assert self.eid is not None
+ self._schemacfg_idx = {}
if session is None:
_session = self.repo.internal_session()
else:
_session = session
try:
- for rql, struct in [('Any ETN WHERE S cw_support ET, ET name ETN, ET is CWEType, S eid %(s)s',
- self.support_entities),
- ('Any RTN WHERE S cw_support RT, RT name RTN, RT is CWRType, S eid %(s)s',
- self.support_relations)]:
- for ertype, in _session.execute(rql, {'s': self.eid}):
- struct[ertype] = True # XXX write support
- for rql, struct in [('Any RTN WHERE S cw_may_cross RT, RT name RTN, S eid %(s)s',
- self.cross_relations),
- ('Any RTN WHERE S cw_dont_cross RT, RT name RTN, S eid %(s)s',
- self.dont_cross_relations)]:
- for rtype, in _session.execute(rql, {'s': self.eid}):
- struct.add(rtype)
+ for schemacfg in _session.execute(
+ 'Any CFG,CFGO,SN,S WHERE '
+ 'CFG options CFGO, CFG cw_schema S, S name SN, '
+ 'CFG cw_for_source X, X eid %(x)s', {'x': self.eid}).entities():
+ self.add_schema_config(schemacfg)
finally:
if session is None:
_session.close()
- # XXX move in hooks or schema constraints
- for rtype in ('is', 'is_instance_of', 'cw_source'):
- assert rtype not in self.dont_cross_relations, \
- '%s relation should not be in dont_cross_relations' % rtype
- assert rtype not in self.support_relations, \
- '%s relation should not be in support_relations' % rtype
+
+ etype_options = set(('write',))
+ rtype_options = set(('maycross', 'dontcross', 'write',))
+
+ def _check_options(self, schemacfg, allowedoptions):
+ if schemacfg.options:
+ options = set(w.strip() for w in schemacfg.options.split(':'))
+ else:
+ options = set()
+ if options - allowedoptions:
+ options = ', '.join(sorted(options - allowedoptions))
+ msg = _('unknown option(s): %s' % options)
+ raise ValidationError(schemacfg.eid, {role_name('options', 'subject'): msg})
+ return options
+
+ def add_schema_config(self, schemacfg, checkonly=False):
+ """added CWSourceSchemaConfig, modify mapping accordingly"""
+ try:
+ ertype = schemacfg.schema.name
+ except AttributeError:
+ msg = schemacfg._cw._("attribute/relation can't be mapped, only "
+ "entity and relation types")
+ raise ValidationError(schemacfg.eid, {role_name('cw_for_schema', 'subject'): msg})
+ if schemacfg.schema.__regid__ == 'CWEType':
+ options = self._check_options(schemacfg, self.etype_options)
+ if not checkonly:
+ self.support_entities[ertype] = 'write' in options
+ else: # CWRType
+ if ertype in ('is', 'is_instance_of', 'cw_source') or ertype in VIRTUAL_RTYPES:
+ msg = schemacfg._cw._('%s relation should not be in mapped') % rtype
+ raise ValidationError(schemacfg.eid, {role_name('cw_for_schema', 'subject'): msg})
+ options = self._check_options(schemacfg, self.rtype_options)
+ if 'dontcross' in options:
+ if 'maycross' in options:
+ msg = schemacfg._("can't mix dontcross and maycross options")
+ raise ValidationError(schemacfg.eid, {role_name('options', 'subject'): msg})
+ if 'write' in options:
+ msg = schemacfg._("can't mix dontcross and write options")
+ raise ValidationError(schemacfg.eid, {role_name('options', 'subject'): msg})
+ if not checkonly:
+ self.dont_cross_relations.add(ertype)
+ elif not checkonly:
+ self.support_relations[ertype] = 'write' in options
+ if 'maycross' in options:
+ self.cross_relations.add(ertype)
+ if not checkonly:
+ # add to an index to ease deletion handling
+ self._schemacfg_idx[schemacfg.eid] = ertype
+
+ def del_schema_config(self, schemacfg, checkonly=False):
+ """deleted CWSourceSchemaConfig, modify mapping accordingly"""
+ if checkonly:
+ return
+ try:
+ ertype = self._schemacfg_idx[schemacfg.eid]
+ if ertype[0].isupper():
+ del self.support_entities[ertype]
+ else:
+ if ertype in self.support_relations:
+ del self.support_relations[ertype]
+ if ertype in self.cross_relations:
+ self.cross_relations.remove(ertype)
+ else:
+ self.dont_cross_relations.remove(ertype)
+ except:
+ self.error('while updating mapping consequently to removal of %s',
+ schemacfg)
def local_eid(self, cnx, extid, session):
etype, dexturi, dextid = cnx.describe(extid)
--- a/server/test/unittest_msplanner.py Mon Feb 07 15:13:05 2011 +0100
+++ b/server/test/unittest_msplanner.py Mon Feb 07 18:19:36 2011 +0100
@@ -63,7 +63,7 @@
{'X': 'CWConstraint'}, {'X': 'CWConstraintType'}, {'X': 'CWEType'},
{'X': 'CWGroup'}, {'X': 'CWPermission'}, {'X': 'CWProperty'},
{'X': 'CWRType'}, {'X': 'CWRelation'},
- {'X': 'CWSource'}, {'X': 'CWSourceHostConfig'},
+ {'X': 'CWSource'}, {'X': 'CWSourceHostConfig'}, {'X': 'CWSourceSchemaConfig'},
{'X': 'CWUser'}, {'X': 'CWUniqueTogetherConstraint'},
{'X': 'Card'}, {'X': 'Comment'}, {'X': 'Division'},
{'X': 'Email'}, {'X': 'EmailAddress'}, {'X': 'EmailPart'},
@@ -897,6 +897,7 @@
ueid = self.session.user.eid
ALL_SOLS = X_ALL_SOLS[:]
ALL_SOLS.remove({'X': 'CWSourceHostConfig'}) # not authorized
+ ALL_SOLS.remove({'X': 'CWSourceSchemaConfig'}) # not authorized
self._test('Any MAX(X)',
[('FetchStep', [('Any E WHERE E type "X", E is Note', [{'E': 'Note'}])],
[self.cards, self.system], None, {'E': 'table1.C0'}, []),
@@ -947,7 +948,7 @@
ueid = self.session.user.eid
X_ET_ALL_SOLS = []
for s in X_ALL_SOLS:
- if s == {'X': 'CWSourceHostConfig'}:
+ if s in ({'X': 'CWSourceHostConfig'}, {'X': 'CWSourceSchemaConfig'}):
continue # not authorized
ets = {'ET': 'CWEType'}
ets.update(s)
@@ -2539,7 +2540,7 @@
None, {'X': 'table0.C0'}, []),
('UnionStep', None, None,
[('OneFetchStep',
- [(u'Any X WHERE X owned_by U, U login "anon", U is CWUser, X is IN(Affaire, BaseTransition, Basket, Bookmark, CWAttribute, CWCache, CWConstraint, CWConstraintType, CWEType, CWGroup, CWPermission, CWProperty, CWRType, CWRelation, CWSource, CWSourceHostConfig, CWUniqueTogetherConstraint, CWUser, Division, Email, EmailAddress, EmailPart, EmailThread, ExternalUri, File, Folder, Personne, RQLExpression, Societe, SubDivision, SubWorkflowExitPoint, Tag, TrInfo, Transition, Workflow, WorkflowTransition)',
+ [(u'Any X WHERE X owned_by U, U login "anon", U is CWUser, X is IN(Affaire, BaseTransition, Basket, Bookmark, CWAttribute, CWCache, CWConstraint, CWConstraintType, CWEType, CWGroup, CWPermission, CWProperty, CWRType, CWRelation, CWSource, CWSourceHostConfig, CWSourceSchemaConfig, CWUniqueTogetherConstraint, CWUser, Division, Email, EmailAddress, EmailPart, EmailThread, ExternalUri, File, Folder, Personne, RQLExpression, Societe, SubDivision, SubWorkflowExitPoint, Tag, TrInfo, Transition, Workflow, WorkflowTransition)',
[{'U': 'CWUser', 'X': 'Affaire'},
{'U': 'CWUser', 'X': 'BaseTransition'},
{'U': 'CWUser', 'X': 'Basket'},
@@ -2556,6 +2557,7 @@
{'U': 'CWUser', 'X': 'CWRelation'},
{'U': 'CWUser', 'X': 'CWSource'},
{'U': 'CWUser', 'X': 'CWSourceHostConfig'},
+ {'U': 'CWUser', 'X': 'CWSourceSchemaConfig'},
{'U': 'CWUser', 'X': 'CWUniqueTogetherConstraint'},
{'U': 'CWUser', 'X': 'CWUser'},
{'U': 'CWUser', 'X': 'Division'},
--- a/server/test/unittest_multisources.py Mon Feb 07 15:13:05 2011 +0100
+++ b/server/test/unittest_multisources.py Mon Feb 07 18:19:36 2011 +0100
@@ -1,4 +1,4 @@
-# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
@@ -17,6 +17,7 @@
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
from datetime import datetime, timedelta
+from itertools import repeat
from cubicweb.devtools import TestServerConfiguration, init_test_database
from cubicweb.devtools.testlib import CubicWebTC, refresh_repo
@@ -46,13 +47,9 @@
Connection_close = Connection.close
def add_extern_mapping(source):
- execute = source._cw.execute
- for etype in ('Card', 'Affaire', 'State'):
- assert execute('SET S cw_support ET WHERE ET name %(etype)s, ET is CWEType, S eid %(s)s',
- {'etype': etype, 's': source.eid})
- for rtype in ('in_state', 'documented_by', 'multisource_inlined_rel'):
- assert execute('SET S cw_support RT WHERE RT name %(rtype)s, RT is CWRType, S eid %(s)s',
- {'rtype': rtype, 's': source.eid})
+ source.init_mapping(zip(('Card', 'Affaire', 'State',
+ 'in_state', 'documented_by', 'multisource_inlined_rel'),
+ repeat(u'write')))
def setUpModule(*args):
@@ -63,6 +60,7 @@
repo3, cnx3 = init_test_database(config=cfg2)
src = cnx3.request().create_entity('CWSource', name=u'extern',
type=u'pyrorql', config=EXTERN_SOURCE_CFG)
+ cnx3.commit() # must commit before adding the mapping
add_extern_mapping(src)
cnx3.commit()
@@ -122,6 +120,7 @@
source = self.request().create_entity(
'CWSource', name=unicode(uri), type=u'pyrorql',
config=unicode(config))
+ self.commit() # must commit before adding the mapping
add_extern_mapping(source)
self.commit()
# trigger discovery
--- a/server/test/unittest_querier.py Mon Feb 07 15:13:05 2011 +0100
+++ b/server/test/unittest_querier.py Mon Feb 07 18:19:36 2011 +0100
@@ -501,15 +501,15 @@
[[u'description_format', 12],
[u'description', 13],
[u'name', 15],
- [u'created_by', 40],
- [u'creation_date', 40],
- [u'cw_source', 40],
- [u'cwuri', 40],
- [u'in_basket', 40],
- [u'is', 40],
- [u'is_instance_of', 40],
- [u'modification_date', 40],
- [u'owned_by', 40]])
+ [u'created_by', 41],
+ [u'creation_date', 41],
+ [u'cw_source', 41],
+ [u'cwuri', 41],
+ [u'in_basket', 41],
+ [u'is', 41],
+ [u'is_instance_of', 41],
+ [u'modification_date', 41],
+ [u'owned_by', 41]])
def test_select_aggregat_having_dumb(self):
# dumb but should not raise an error
--- a/test/unittest_schema.py Mon Feb 07 15:13:05 2011 +0100
+++ b/test/unittest_schema.py Mon Feb 07 18:19:36 2011 +0100
@@ -163,7 +163,7 @@
'CWCache', 'CWConstraint', 'CWConstraintType', 'CWEType',
'CWAttribute', 'CWGroup', 'EmailAddress', 'CWRelation',
'CWPermission', 'CWProperty', 'CWRType',
- 'CWSource', 'CWSourceHostConfig',
+ 'CWSource', 'CWSourceHostConfig', 'CWSourceSchemaConfig',
'CWUniqueTogetherConstraint', 'CWUser',
'ExternalUri', 'File', 'Float', 'Int', 'Interval', 'Note',
'Password', 'Personne',
@@ -171,7 +171,7 @@
'Societe', 'State', 'StateFull', 'String', 'SubNote', 'SubWorkflowExitPoint',
'Tag', 'Time', 'Transition', 'TrInfo',
'Workflow', 'WorkflowTransition']
- self.assertListEqual(entities, sorted(expected_entities))
+ self.assertListEqual(sorted(expected_entities), entities)
relations = sorted([str(r) for r in schema.relations()])
expected_relations = ['add_permission', 'address', 'alias', 'allowed_transition',
'bookmarked_by', 'by_transition',
@@ -181,8 +181,7 @@
'constrained_by', 'constraint_of',
'content', 'content_format',
'created_by', 'creation_date', 'cstrtype', 'custom_workflow',
- 'cwuri', 'cw_source', 'cw_host_config_of',
- 'cw_support', 'cw_dont_cross', 'cw_may_cross',
+ 'cwuri', 'cw_for_source', 'cw_host_config_of', 'cw_schema', 'cw_source',
'data', 'data_encoding', 'data_format', 'data_name', 'default_workflow', 'defaultval', 'delete_permission',
'description', 'description_format', 'destination_state',
@@ -202,7 +201,7 @@
'name', 'nom',
- 'ordernum', 'owned_by',
+ 'options', 'ordernum', 'owned_by',
'path', 'pkey', 'prefered_form', 'prenom', 'primary_email',
@@ -218,7 +217,7 @@
'wf_info_for', 'wikiid', 'workflow_of', 'tr_count']
- self.assertListEqual(relations, sorted(expected_relations))
+ self.assertListEqual(sorted(expected_relations), relations)
eschema = schema.eschema('CWUser')
rels = sorted(str(r) for r in eschema.subject_relations())
--- a/web/views/cwsources.py Mon Feb 07 15:13:05 2011 +0100
+++ b/web/views/cwsources.py Mon Feb 07 18:19:36 2011 +0100
@@ -1,4 +1,4 @@
-# copyright 2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# copyright 2010-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
#
# This file is part of CubicWeb.
@@ -15,22 +15,24 @@
#
# You should have received a copy of the GNU Lesser General Public License along
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
-"""Specific views for data sources"""
+"""Specific views for data sources and related entities (eg CWSource,
+CWSourceHostConfig, CWSourceSchemaConfig).
+"""
__docformat__ = "restructuredtext en"
_ = unicode
from itertools import repeat, chain
-from cubicweb.selectors import is_instance, score_entity
+from cubicweb.selectors import is_instance, score_entity, match_user_groups
from cubicweb.view import EntityView
from cubicweb.schema import META_RTYPES, VIRTUAL_RTYPES, display_name
from cubicweb.web import uicfg
from cubicweb.web.views import tabs
-for rtype in ('cw_support', 'cw_may_cross', 'cw_dont_cross'):
- uicfg.primaryview_section.tag_subject_of(('CWSource', rtype, '*'),
- 'hidden')
+_pvs = uicfg.primaryview_section
+_pvs.tag_object_of(('*', 'cw_for_source', 'CWSource'), 'hidden')
+
class CWSourcePrimaryView(tabs.TabbedPrimaryView):
__select__ = is_instance('CWSource')
@@ -43,129 +45,155 @@
__select__ = tabs.PrimaryTab.__select__ & is_instance('CWSource')
+MAPPED_SOURCE_TYPES = set( ('pyrorql', 'datafeed') )
+
class CWSourceMappingTab(EntityView):
__regid__ = 'cwsource-mapping'
__select__ = (tabs.PrimaryTab.__select__ & is_instance('CWSource')
- & score_entity(lambda x:x.type == 'pyrorql'))
+ & match_user_groups('managers')
+ & score_entity(lambda x:x.type in MAPPED_SOURCE_TYPES))
def entity_call(self, entity):
_ = self._cw._
- self.w('<h3>%s</h3>' % _('Entity and relation types supported by this source'))
- self.wview('list', entity.related('cw_support'), 'noresult')
- self.w('<h3>%s</h3>' % _('Relations that should not be crossed'))
- self.w('<p>%s</p>' % _(
- 'By default, when a relation is not supported by a source, it is '
- 'supposed that a local relation may point to an entity from the '
- 'external source. Relations listed here won\'t have this '
- '"crossing" behaviour.'))
- self.wview('list', entity.related('cw_dont_cross'), 'noresult')
- self.w('<h3>%s</h3>' % _('Relations that can be crossed'))
- self.w('<p>%s</p>' % _(
- 'By default, when a relation is supported by a source, it is '
- 'supposed that a local relation can\'t point to an entity from the '
- 'external source. Relations listed here may have this '
- '"crossing" behaviour anyway.'))
- self.wview('list', entity.related('cw_may_cross'), 'noresult')
- if self._cw.user.is_in_group('managers'):
- errors, warnings, infos = check_mapping(entity)
- if (errors or warnings or infos):
+ self.w('<h3>%s</h3>' % _('Entity and relation supported by this source'))
+ rset = self._cw.execute(
+ 'Any X, SCH, XO ORDERBY ET WHERE X options XO, X cw_for_source S, S eid %(s)s, '
+ 'X cw_schema SCH, SCH is ET', {'s': entity.eid})
+ self.wview('table', rset, 'noresult')
+ # self.w('<h3>%s</h3>' % _('Relations that should not be crossed'))
+ # self.w('<p>%s</p>' % _(
+ # 'By default, when a relation is not supported by a source, it is '
+ # 'supposed that a local relation may point to an entity from the '
+ # 'external source. Relations listed here won\'t have this '
+ # '"crossing" behaviour.'))
+ # self.wview('list', entity.related('cw_dont_cross'), 'noresult')
+ # self.w('<h3>%s</h3>' % _('Relations that can be crossed'))
+ # self.w('<p>%s</p>' % _(
+ # 'By default, when a relation is supported by a source, it is '
+ # 'supposed that a local relation can\'t point to an entity from the '
+ # 'external source. Relations listed here may have this '
+ # '"crossing" behaviour anyway.'))
+ # self.wview('list', entity.related('cw_may_cross'), 'noresult')
+ checker = MAPPING_CHECKERS.get(entity.type, MappingChecker)(entity)
+ checker.check()
+ if (checker.errors or checker.warnings or checker.infos):
self.w('<h2>%s</h2>' % _('Detected problems'))
- errors = zip(repeat(_('error'), errors))
- warnings = zip(repeat(_('warning'), warnings))
- infos = zip(repeat(_('warning'), infos))
+ errors = zip(repeat(_('error')), checker.errors)
+ warnings = zip(repeat(_('warning')), checker.warnings)
+ infos = zip(repeat(_('warning')), checker.infos)
self.wview('pyvaltable', pyvalue=chain(errors, warnings, infos))
-def check_mapping(cwsource):
- req = cwsource._cw
- _ = req._
- errors = []
- error = errors.append
- warnings = []
- warning = warnings.append
- infos = []
- info = infos.append
- srelations = set()
- sentities = set()
- maycross = set()
- dontcross = set()
- # first check supported stuff / meta & virtual types and get mapping as sets
- for cwertype in cwsource.cw_support:
- if cwertype.name in META_RTYPES:
- error(_('meta relation %s can not be supported') % cwertype.name)
- else:
- if cwertype.__regid__ == 'CWEType':
- sentities.add(cwertype.name)
- else:
- srelations.add(cwertype.name)
- for attr, attrset in (('cw_may_cross', maycross),
- ('cw_dont_cross', dontcross)):
- for cwrtype in getattr(cwsource, attr):
- if cwrtype.name in VIRTUAL_RTYPES:
- error(_('virtual relation %(rtype)s can not be referenced by '
- 'the "%(srel)s" relation') %
- {'rtype': cwrtype.name,
- 'srel': display_name(req, attr, context='CWSource')})
+
+class MappingChecker(object):
+ def __init__(self, cwsource):
+ self.cwsource = cwsource
+ self.errors = []
+ self.warnings = []
+ self.infos = []
+ self.schema = cwsource._cw.vreg.schema
+
+ def init(self):
+ # supported entity types
+ self.sentities = set()
+ # supported relations
+ self.srelations = {}
+ # avoid duplicated messages
+ self.seen = set()
+ # first get mapping as dict/sets
+ for schemacfg in self.cwsource.reverse_cw_for_source:
+ self.init_schemacfg(schemacfg)
+
+ def init_schemacfg(self, schemacfg):
+ cwerschema = schemacfg.schema
+ if cwerschema.__regid__ == 'CWEType':
+ self.sentities.add(cwerschema.name)
+ elif cwerschema.__regid__ == 'CWRType':
+ assert not cwerschema.name in self.srelations
+ self.srelations[cwerschema.name] = None
+ else: # CWAttribute/CWRelation
+ self.srelations.setdefault(cwerschema.rtype.name, []).append(
+ (cwerschema.stype.name, cwerschema.otype.name) )
+
+ def check(self):
+ self.init()
+ error = self.errors.append
+ warning = self.warnings.append
+ info = self.infos.append
+ for etype in self.sentities:
+ eschema = self.schema[etype]
+ for rschema, ttypes, role in eschema.relation_definitions():
+ if rschema in META_RTYPES:
+ continue
+ ttypes = [ttype for ttype in ttypes if ttype in self.sentities]
+ if not rschema in self.srelations:
+ for ttype in ttypes:
+ rdef = rschema.role_rdef(etype, ttype, role)
+ self.seen.add(rdef)
+ if rdef.role_cardinality(role) in '1+':
+ error(_('relation %(type)s with %(etype)s as %(role)s '
+ 'and target type %(target)s is mandatory but '
+ 'not supported') %
+ {'rtype': rschema, 'etype': etype, 'role': role,
+ 'target': ttype})
+ elif ttype in self.sentities:
+ warning(_('%s could be supported') % rdef)
+ elif not ttypes:
+ warning(_('relation %(rtype)s with %(etype)s as %(role)s is '
+ 'supported but no target type supported') %
+ {'rtype': rschema, 'role': role, 'etype': etype})
+ for rtype in self.srelations:
+ rschema = self.schema[rtype]
+ for subj, obj in rschema.rdefs:
+ if subj in self.sentities and obj in self.sentities:
+ break
else:
- attrset.add(cwrtype.name)
- # check relation in dont_cross_relations aren't in support_relations
- for rtype in dontcross & maycross:
- info(_('relation %(rtype)s is supported but in %(dontcross)s') %
- {'rtype': rtype,
- 'dontcross': display_name(req, 'cw_dont_cross',
- context='CWSource')})
- # check relation in cross_relations are in support_relations
- for rtype in maycross & srelations:
- info(_('relation %(rtype)s isn\'t supported but in %(maycross)s') %
- {'rtype': rtype,
- 'dontcross': display_name(req, 'cw_may_cross',
- context='CWSource')})
- # now check for more handy things
- seen = set()
- for etype in sentities:
- eschema = req.vreg.schema[etype]
- for rschema, ttypes, role in eschema.relation_definitions():
- if rschema in META_RTYPES:
- continue
- ttypes = [ttype for ttype in ttypes if ttype in sentities]
- if not rschema in srelations:
- somethingprinted = False
- for ttype in ttypes:
- rdef = rschema.role_rdef(etype, ttype, role)
- seen.add(rdef)
- if rdef.role_cardinality(role) in '1+':
- error(_('relation %(type)s with %(etype)s as %(role)s '
- 'and target type %(target)s is mandatory but '
- 'not supported') %
- {'rtype': rschema, 'etype': etype, 'role': role,
- 'target': ttype})
- somethingprinted = True
- elif ttype in sentities:
- if rdef not in seen:
- warning(_('%s could be supported') % rdef)
- somethingprinted = True
- if rschema not in dontcross:
- if role == 'subject' and rschema.inlined:
- error(_('inlined relation %(rtype)s of %(etype)s '
- 'should be supported') %
- {'rtype': rschema, 'etype': etype})
- elif (not somethingprinted and rschema not in seen
- and rschema not in maycross):
- info(_('you may want to specify something for %s') %
- rschema)
- seen.add(rschema)
- else:
- if not ttypes:
- warning(_('relation %(rtype)s with %(etype)s as %(role)s '
- 'is supported but no target type supported') %
- {'rtype': rschema, 'role': role, 'etype': etype})
- if rschema in maycross and rschema.inlined:
+ error(_('relation %s is supported but none if its definitions '
+ 'matches supported entities') % rtype)
+ self.custom_check()
+
+ def custom_check(self):
+ pass
+
+
+class PyroRQLMappingChecker(MappingChecker):
+ """pyrorql source mapping checker"""
+
+ def init(self):
+ self.dontcross = set()
+ self.maycross = set()
+ super(PyroRQLMappingChecker, self).init()
+
+ def init_schemacfg(self, schemacfg):
+ options = schemacfg.options or ()
+ if 'dontcross' in options:
+ self.dontcross.add(schemacfg.schema.name)
+ else:
+ super(PyroRQLMappingChecker, self).init_schemacfg(schemacfg)
+ if 'maycross' in options:
+ self.maycross.add(schemacfg.schema.name)
+
+ def custom_check(self):
+ error = self.errors.append
+ info = self.infos.append
+ for etype in self.sentities:
+ eschema = self.schema[etype]
+ for rschema, ttypes, role in eschema.relation_definitions():
+ if rschema in META_RTYPES:
+ continue
+ if not rschema in self.srelations:
+ if rschema not in self.dontcross:
+ if role == 'subject' and rschema.inlined:
+ error(_('inlined relation %(rtype)s of %(etype)s '
+ 'should be supported') %
+ {'rtype': rschema, 'etype': etype})
+ elif (rschema not in self.seen and rschema not in self.maycross):
+ info(_('you may want to specify something for %s') %
+ rschema)
+ self.seen.add(rschema)
+ elif rschema in self.maycross and rschema.inlined:
error(_('you should un-inline relation %s which is '
'supported and may be crossed ') % rschema)
- for rschema in srelations:
- for subj, obj in rschema.rdefs:
- if subj in sentities and obj in sentities:
- break
- else:
- error(_('relation %s is supported but none if its definitions '
- 'matches supported entities') % rschema)
- return errors, warnings, infos
+
+MAPPING_CHECKERS = {
+ 'pyrorql': PyroRQLMappingChecker,
+ }