# HG changeset patch # User Sylvain Thénault # Date 1291219895 -3600 # Node ID 24bf6f181d0e4b6715f46448b1a0d676d5c2cd95 # Parent a2ccbcbb08a6e4d6ec4447e313f5be1135edaf34 [pyro source] store pyro source mapping file into the database also, remove former c-c command to check that file and do the job in the source's view. diff -r a2ccbcbb08a6 -r 24bf6f181d0e __pkginfo__.py --- a/__pkginfo__.py Wed Dec 01 17:09:19 2010 +0100 +++ b/__pkginfo__.py Wed Dec 01 17:11:35 2010 +0100 @@ -22,7 +22,7 @@ modname = distname = "cubicweb" -numversion = (3, 10, 6) +numversion = (3, 11, 0) version = '.'.join(str(num) for num in numversion) description = "a repository of entities / relations for knowledge management" diff -r a2ccbcbb08a6 -r 24bf6f181d0e hooks/syncsources.py --- a/hooks/syncsources.py Wed Dec 01 17:09:19 2010 +0100 +++ b/hooks/syncsources.py Wed Dec 01 17:11:35 2010 +0100 @@ -1,6 +1,7 @@ +from yams.schema import role_name from cubicweb import ValidationError from cubicweb.selectors import is_instance -from cubicweb.server import hook +from cubicweb.server import SOURCE_TYPES, hook class SourceHook(hook.Hook): __abstract__ = True @@ -8,7 +9,7 @@ class SourceAddedOp(hook.Operation): - def precommit_event(self): + def postcommit_event(self): self.session.repo.add_source(self.entity) class SourceAddedHook(SourceHook): @@ -16,6 +17,10 @@ __select__ = SourceHook.__select__ & is_instance('CWSource') events = ('after_add_entity',) def __call__(self): + if not self.entity.type in SOURCE_TYPES: + msg = self._cw._('unknown source type') + raise ValidationError(self.entity.eid, + {role_name('type', 'subject'): msg}) SourceAddedOp(self._cw, entity=self.entity) @@ -31,3 +36,13 @@ if self.entity.name == 'system': raise ValidationError(self.entity.eid, {None: 'cant remove system source'}) SourceRemovedOp(self._cw, uri=self.entity.name) + +class SourceRemovedHook(SourceHook): + __regid__ = 'cw.sources.removed' + __select__ = SourceHook.__select__ & hook.match_rtype('cw_support', 'cw_may_cross') + events = ('after_add_relation',) + def __call__(self): + entity = self._cw.entity_from_eid(self.eidto) + if entity.__regid__ == 'CWRType' and entity.name in ('is', 'is_instance_of', 'cw_source'): + msg = self._cw._('the %s relation type can\'t be used here') % entity.name + raise ValidationError(self.eidto, {role_name(self.rtype, 'subject'): msg}) diff -r a2ccbcbb08a6 -r 24bf6f181d0e misc/migration/3.11.0_Any.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/misc/migration/3.11.0_Any.py Wed Dec 01 17:11:35 2010 +0100 @@ -0,0 +1,57 @@ +sync_schema_props_perms('cw_support', syncperms=False) +sync_schema_props_perms('cw_dont_cross', syncperms=False) +sync_schema_props_perms('cw_may_cross', syncperms=False) + +try: + from cubicweb.server.sources.pyrorql import PyroRQLSource +except ImportError: + pass +else: + + from os.path import join + + def load_mapping_file(source): + mappingfile = source.config['mapping-file'] + mappingfile = join(source.repo.config.apphome, mappingfile) + mapping = {} + execfile(mappingfile, mapping) + for junk in ('__builtins__', '__doc__'): + mapping.pop(junk, None) + mapping.setdefault('support_relations', {}) + mapping.setdefault('dont_cross_relations', set()) + mapping.setdefault('cross_relations', set()) + # do some basic checks of the mapping content + assert 'support_entities' in mapping, \ + 'mapping file should at least define support_entities' + assert isinstance(mapping['support_entities'], dict) + assert isinstance(mapping['support_relations'], dict) + assert isinstance(mapping['dont_cross_relations'], set) + assert isinstance(mapping['cross_relations'], set) + unknown = set(mapping) - set( ('support_entities', 'support_relations', + 'dont_cross_relations', 'cross_relations') ) + assert not unknown, 'unknown mapping attribute(s): %s' % unknown + # relations that are necessarily not crossed + for rtype in ('is', 'is_instance_of', 'cw_source'): + assert rtype not in mapping['dont_cross_relations'], \ + '%s relation should not be in dont_cross_relations' % rtype + assert rtype not in mapping['support_relations'], \ + '%s relation should not be in support_relations' % rtype + return mapping + + for source in repo.sources_by_uri.values(): + if not isinstance(source, PyroRQLSource): + continue + mapping = load_mapping_file(source) + print 'migrating map for', source + for etype in mapping['support_entities']: # XXX write support + rql('SET S cw_support ET WHERE ET name %(etype)s, ET is CWEType, S eid %(s)s', + {'etype': etype, 's': source.eid}) + for rtype in mapping['support_relations']: # XXX write support + rql('SET S cw_support RT WHERE RT name %(rtype)s, RT is CWRType, S eid %(s)s', + {'rtype': rtype, 's': source.eid}) + for rtype in mapping['dont_cross_relations']: # XXX write support + rql('SET S cw_dont_cross RT WHERE RT name %(rtype)s, S eid %(s)s', + {'rtype': rtype, 's': source.eid}) + for rtype in mapping['cross_relations']: # XXX write support + rql('SET S cw_may_cross RT WHERE RT name %(rtype)s, S eid %(s)s', + {'rtype': rtype, 's': source.eid}) diff -r a2ccbcbb08a6 -r 24bf6f181d0e schemas/base.py --- a/schemas/base.py Wed Dec 01 17:09:19 2010 +0100 +++ b/schemas/base.py Wed Dec 01 17:11:35 2010 +0100 @@ -295,14 +295,19 @@ class cw_support(RelationDefinition): subject = 'CWSource' object = ('CWEType', 'CWRType') + constraints = [RQLConstraint('NOT O final TRUE')] class cw_dont_cross(RelationDefinition): subject = 'CWSource' object = 'CWRType' + constraints = [RQLConstraint('NOT O final TRUE'), + RQLConstraint('NOT S cw_may_cross O')] class cw_may_cross(RelationDefinition): subject = 'CWSource' object = 'CWRType' + constraints = [RQLConstraint('NOT O final TRUE'), + RQLConstraint('NOT S cw_dont_cross O')] # "abtract" relation types, no definition in cubicweb itself ################### diff -r a2ccbcbb08a6 -r 24bf6f181d0e server/checkintegrity.py --- a/server/checkintegrity.py Wed Dec 01 17:09:19 2010 +0100 +++ b/server/checkintegrity.py Wed Dec 01 17:11:35 2010 +0100 @@ -19,8 +19,6 @@ * integrity of a CubicWeb repository. Hum actually only the system database is checked. - -* consistency of multi-sources instance mapping file """ from __future__ import with_statement @@ -32,7 +30,7 @@ from logilab.common.shellutils import ProgressBar -from cubicweb.schema import META_RTYPES, VIRTUAL_RTYPES, PURE_VIRTUAL_RTYPES +from cubicweb.schema import PURE_VIRTUAL_RTYPES from cubicweb.server.sqlutils import SQL_PREFIX from cubicweb.server.session import security_enabled @@ -334,103 +332,3 @@ session.set_pool() reindex_entities(repo.schema, session, withpb=withpb) cnx.commit() - - -def info(msg, *args): - if args: - msg = msg % args - print 'INFO: %s' % msg - -def warning(msg, *args): - if args: - msg = msg % args - print 'WARNING: %s' % msg - -def error(msg, *args): - if args: - msg = msg % args - print 'ERROR: %s' % msg - -def check_mapping(schema, mapping, warning=warning, error=error): - # first check stuff found in mapping file exists in the schema - for attr in ('support_entities', 'support_relations'): - for ertype in mapping[attr].keys(): - try: - mapping[attr][ertype] = erschema = schema[ertype] - except KeyError: - error('reference to unknown type %s in %s', ertype, attr) - del mapping[attr][ertype] - else: - if erschema.final or erschema in META_RTYPES: - error('type %s should not be mapped in %s', ertype, attr) - del mapping[attr][ertype] - for attr in ('dont_cross_relations', 'cross_relations'): - for rtype in list(mapping[attr]): - try: - rschema = schema.rschema(rtype) - except KeyError: - error('reference to unknown relation type %s in %s', rtype, attr) - mapping[attr].remove(rtype) - else: - if rschema.final or rschema in VIRTUAL_RTYPES: - error('relation type %s should not be mapped in %s', - rtype, attr) - mapping[attr].remove(rtype) - # check relation in dont_cross_relations aren't in support_relations - for rschema in mapping['dont_cross_relations']: - if rschema in mapping['support_relations']: - info('relation %s is in dont_cross_relations and in support_relations', - rschema) - # check relation in cross_relations are in support_relations - for rschema in mapping['cross_relations']: - if rschema not in mapping['support_relations']: - info('relation %s is in cross_relations but not in support_relations', - rschema) - # check for relation in both cross_relations and dont_cross_relations - for rschema in mapping['cross_relations'] & mapping['dont_cross_relations']: - error('relation %s is in both cross_relations and dont_cross_relations', - rschema) - # now check for more handy things - seen = set() - for eschema in mapping['support_entities'].values(): - for rschema, ttypes, role in eschema.relation_definitions(): - if rschema in META_RTYPES: - continue - ttypes = [ttype for ttype in ttypes if ttype in mapping['support_entities']] - if not rschema in mapping['support_relations']: - somethingprinted = False - for ttype in ttypes: - rdef = rschema.role_rdef(eschema, ttype, role) - seen.add(rdef) - if rdef.role_cardinality(role) in '1+': - error('relation %s with %s as %s and target type %s is ' - 'mandatory but not supported', - rschema, eschema, role, ttype) - somethingprinted = True - elif ttype in mapping['support_entities']: - if rdef not in seen: - warning('%s could be supported', rdef) - somethingprinted = True - if rschema not in mapping['dont_cross_relations']: - if role == 'subject' and rschema.inlined: - error('inlined relation %s of %s should be supported', - rschema, eschema) - elif not somethingprinted and rschema not in seen and rschema not in mapping['cross_relations']: - print 'you may want to specify something for %s' % rschema - seen.add(rschema) - else: - if not ttypes: - warning('relation %s with %s as %s is supported but no target ' - 'type supported', rschema, role, eschema) - if rschema in mapping['cross_relations'] and rschema.inlined: - error('you should unline relation %s which is supported and ' - 'may be crossed ', rschema) - for rschema in mapping['support_relations'].values(): - if rschema in META_RTYPES: - continue - for subj, obj in rschema.rdefs: - if subj in mapping['support_entities'] and obj in mapping['support_entities']: - break - else: - error('relation %s is supported but none if its definitions ' - 'matches supported entities', rschema) diff -r a2ccbcbb08a6 -r 24bf6f181d0e server/repository.py --- a/server/repository.py Wed Dec 01 17:09:19 2010 +0100 +++ b/server/repository.py Wed Dec 01 17:11:35 2010 +0100 @@ -211,8 +211,8 @@ # needed (for instance looking for persistent configuration using an # internal session, which is not possible until pools have been # initialized) - for source in self.sources: - source.init() + for source in self.sources_by_uri.itervalues(): + source.init(source in self.sources) else: # call init_creating so that for instance native source can # configurate tsearch according to postgres version @@ -263,11 +263,14 @@ self.sources_by_eid[sourceent.eid] = source self.sources_by_uri[sourceent.name] = source if self.config.source_enabled(source): + source.init(True, session=sourceent._cw) self.sources.append(source) self.querier.set_planner() if add_to_pools: for pool in self.pools: pool.add_source(source) + else: + source.init(False, session=sourceent._cw) self._clear_planning_caches() def remove_source(self, uri): diff -r a2ccbcbb08a6 -r 24bf6f181d0e server/serverctl.py --- a/server/serverctl.py Wed Dec 01 17:09:19 2010 +0100 +++ b/server/serverctl.py Wed Dec 01 17:11:35 2010 +0100 @@ -897,39 +897,11 @@ mih.cmd_synchronize_schema() -class CheckMappingCommand(Command): - """Check content of the mapping file of an external source. - - The mapping is checked against the instance's schema, searching for - inconsistencies or stuff you may have forgotten. It's higly recommanded to - run it when you setup a multi-sources instance. - - - the identifier of the instance. - - - the mapping file to check. - """ - name = 'check-mapping' - arguments = ' ' - min_args = max_args = 2 - - def run(self, args): - from cubicweb.server.checkintegrity import check_mapping - from cubicweb.server.sources.pyrorql import load_mapping_file - appid, mappingfile = args - config = ServerConfiguration.config_for(appid) - config.quick_start = True - mih = config.migration_handler(connect=False, verbosity=1) - repo = mih.repo_connect() # necessary to get cubes - check_mapping(config.load_schema(), load_mapping_file(mappingfile)) - for cmdclass in (CreateInstanceDBCommand, InitInstanceCommand, GrantUserOnInstanceCommand, ResetAdminPasswordCommand, StartRepositoryCommand, DBDumpCommand, DBRestoreCommand, DBCopyCommand, AddSourceCommand, CheckRepositoryCommand, RebuildFTICommand, SynchronizeInstanceSchemaCommand, - CheckMappingCommand, ): CWCTL.register(cmdclass) diff -r a2ccbcbb08a6 -r 24bf6f181d0e server/sources/__init__.py --- a/server/sources/__init__.py Wed Dec 01 17:09:19 2010 +0100 +++ b/server/sources/__init__.py Wed Dec 01 17:11:35 2010 +0100 @@ -116,8 +116,10 @@ """method called by the repository once ready to create a new instance""" pass - def init(self): - """method called by the repository once ready to handle request""" + def init(self, activated, session=None): + """method called by the repository once ready to handle request. + `activated` is a boolean flag telling if the source is activated or not. + """ pass def backup(self, backupfile, confirm): @@ -146,7 +148,7 @@ pass def __repr__(self): - return '<%s source @%#x>' % (self.uri, id(self)) + return '<%s source %s @%#x>' % (self.uri, self.eid, id(self)) def __cmp__(self, other): """simple comparison function to get predictable source order, with the diff -r a2ccbcbb08a6 -r 24bf6f181d0e server/sources/ldapuser.py --- a/server/sources/ldapuser.py Wed Dec 01 17:09:19 2010 +0100 +++ b/server/sources/ldapuser.py Wed Dec 01 17:11:35 2010 +0100 @@ -199,14 +199,15 @@ self._cache = {} self._query_cache = TimedCache(self._cache_ttl) - def init(self): + def init(self, activated, session=None): """method called by the repository once ready to handle request""" - self.info('ldap init') - # set minimum period of 5min 1s (the additional second is to minimize - # resonnance effet) - self.repo.looping_task(max(301, self._interval), self.synchronize) - self.repo.looping_task(self._cache_ttl // 10, - self._query_cache.clear_expired) + if activated: + self.info('ldap init') + # set minimum period of 5min 1s (the additional second is to + # minimize resonnance effet) + self.repo.looping_task(max(301, self._interval), self.synchronize) + self.repo.looping_task(self._cache_ttl // 10, + self._query_cache.clear_expired) def synchronize(self): """synchronize content known by this repository with content in the diff -r a2ccbcbb08a6 -r 24bf6f181d0e server/sources/native.py --- a/server/sources/native.py Wed Dec 01 17:09:19 2010 +0100 +++ b/server/sources/native.py Wed Dec 01 17:11:35 2010 +0100 @@ -326,17 +326,21 @@ """execute the query and return its result""" return self.process_result(self.doexec(session, sql, args)) - def init_creating(self): - pool = self.repo._get_pool() - pool.pool_set() + def init_creating(self, pool=None): # check full text index availibility if self.do_fti: - if not self.dbhelper.has_fti_table(pool['system']): + if pool is None: + _pool = self.repo._get_pool() + _pool.pool_set() + else: + _pool = pool + if not self.dbhelper.has_fti_table(_pool['system']): if not self.repo.config.creating: self.critical('no text index table') self.do_fti = False - pool.pool_reset() - self.repo._free_pool(pool) + if pool is None: + _pool.pool_reset() + self.repo._free_pool(_pool) def backup(self, backupfile, confirm): """method called to create a backup of the source's data""" @@ -356,8 +360,8 @@ if self.repo.config.open_connections_pools: self.open_pool_connections() - def init(self): - self.init_creating() + def init(self, activated, session=None): + self.init_creating(session and session.pool) def shutdown(self): if self._eid_creation_cnx: diff -r a2ccbcbb08a6 -r 24bf6f181d0e server/sources/pyrorql.py --- a/server/sources/pyrorql.py Wed Dec 01 17:09:19 2010 +0100 +++ b/server/sources/pyrorql.py Wed Dec 01 17:11:35 2010 +0100 @@ -45,34 +45,6 @@ select, col = union.locate_subquery(col, etype, args) return getattr(select.selection[col], 'uidtype', None) -def load_mapping_file(mappingfile): - mapping = {} - execfile(mappingfile, mapping) - for junk in ('__builtins__', '__doc__'): - mapping.pop(junk, None) - mapping.setdefault('support_relations', {}) - mapping.setdefault('dont_cross_relations', set()) - mapping.setdefault('cross_relations', set()) - - # do some basic checks of the mapping content - assert 'support_entities' in mapping, \ - 'mapping file should at least define support_entities' - assert isinstance(mapping['support_entities'], dict) - assert isinstance(mapping['support_relations'], dict) - assert isinstance(mapping['dont_cross_relations'], set) - assert isinstance(mapping['cross_relations'], set) - unknown = set(mapping) - set( ('support_entities', 'support_relations', - 'dont_cross_relations', 'cross_relations') ) - assert not unknown, 'unknown mapping attribute(s): %s' % unknown - # relations that are necessarily not crossed - mapping['dont_cross_relations'] |= set(('owned_by', 'created_by')) - for rtype in ('is', 'is_instance_of', 'cw_source'): - assert rtype not in mapping['dont_cross_relations'], \ - '%s relation should not be in dont_cross_relations' % rtype - assert rtype not in mapping['support_relations'], \ - '%s relation should not be in support_relations' % rtype - return mapping - class ReplaceByInOperator(Exception): def __init__(self, eids): @@ -96,12 +68,6 @@ 'help': 'identifier of the repository in the pyro name server', 'group': 'pyro-source', 'level': 0, }), - ('mapping-file', - {'type' : 'string', - 'default': REQUIRED, - 'help': 'path to a python file with the schema mapping definition', - 'group': 'pyro-source', 'level': 1, - }), ('cubicweb-user', {'type' : 'string', 'default': REQUIRED, @@ -156,24 +122,7 @@ def __init__(self, repo, source_config, *args, **kwargs): AbstractSource.__init__(self, repo, source_config, *args, **kwargs) - mappingfile = source_config['mapping-file'] - if not mappingfile[0] == '/': - mappingfile = join(repo.config.apphome, mappingfile) - try: - mapping = load_mapping_file(mappingfile) - except IOError: - self.disabled = True - self.error('cant read mapping file %s, source disabled', - mappingfile) - self.support_entities = {} - self.support_relations = {} - self.dont_cross_relations = set() - self.cross_relations = set() - else: - self.support_entities = mapping['support_entities'] - self.support_relations = mapping['support_relations'] - self.dont_cross_relations = mapping['dont_cross_relations'] - self.cross_relations = mapping['cross_relations'] + # XXX get it through pyro if unset baseurl = source_config.get('base-url') if baseurl and not baseurl.endswith('/'): source_config['base-url'] += '/' @@ -212,12 +161,47 @@ finally: session.close() - def init(self): + def init(self, activated, session=None): """method called by the repository once ready to handle request""" - interval = int(self.config.get('synchronization-interval', 5*60)) - self.repo.looping_task(interval, self.synchronize) - self.repo.looping_task(self._query_cache.ttl.seconds/10, - self._query_cache.clear_expired) + self.load_mapping(session) + if activated: + interval = int(self.config.get('synchronization-interval', 5*60)) + self.repo.looping_task(interval, self.synchronize) + self.repo.looping_task(self._query_cache.ttl.seconds/10, + self._query_cache.clear_expired) + + def load_mapping(self, session=None): + self.support_entities = {} + self.support_relations = {} + self.dont_cross_relations = set(('owned_by', 'created_by')) + self.cross_relations = set() + assert self.eid is not None + if session is None: + _session = self.repo.internal_session() + else: + _session = session + try: + for rql, struct in [('Any ETN WHERE S cw_support ET, ET name ETN, ET is CWEType, S eid %(s)s', + self.support_entities), + ('Any RTN WHERE S cw_support RT, RT name RTN, RT is CWRType, S eid %(s)s', + self.support_relations)]: + for ertype, in _session.execute(rql, {'s': self.eid}): + struct[ertype] = True # XXX write support + for rql, struct in [('Any RTN WHERE S cw_may_cross RT, RT name RTN, S eid %(s)s', + self.cross_relations), + ('Any RTN WHERE S cw_dont_cross RT, RT name RTN, S eid %(s)s', + self.dont_cross_relations)]: + for rtype, in _session.execute(rql, {'s': self.eid}): + struct.add(rtype) + finally: + if session is None: + _session.close() + # XXX move in hooks or schema constraints + for rtype in ('is', 'is_instance_of', 'cw_source'): + assert rtype not in self.dont_cross_relations, \ + '%s relation should not be in dont_cross_relations' % rtype + assert rtype not in self.support_relations, \ + '%s relation should not be in support_relations' % rtype def local_eid(self, cnx, extid, session): etype, dexturi, dextid = cnx.describe(extid) @@ -246,8 +230,8 @@ etypes = self.support_entities.keys() if mtime is None: mtime = self.last_update_time() - updatetime, modified, deleted = extrepo.entities_modified_since(etypes, - mtime) + updatetime, modified, deleted = extrepo.entities_modified_since( + etypes, mtime) self._query_cache.clear() repo = self.repo session = repo.internal_session() diff -r a2ccbcbb08a6 -r 24bf6f181d0e server/test/unittest_multisources.py --- a/server/test/unittest_multisources.py Wed Dec 01 17:09:19 2010 +0100 +++ b/server/test/unittest_multisources.py Wed Dec 01 17:11:35 2010 +0100 @@ -1,4 +1,4 @@ - # copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved. +# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved. # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr # # This file is part of CubicWeb. @@ -35,7 +35,6 @@ pyro-ns-id = extern cubicweb-user = admin cubicweb-password = gingkow -mapping-file = extern_mapping.py base-url=http://extern.org/ ''' @@ -46,14 +45,25 @@ PyroRQLSource_get_connection = PyroRQLSource.get_connection Connection_close = Connection.close +def add_extern_mapping(source): + execute = source._cw.execute + for etype in ('Card', 'Affaire', 'State'): + assert execute('SET S cw_support ET WHERE ET name %(etype)s, ET is CWEType, S eid %(s)s', + {'etype': etype, 's': source.eid}) + for rtype in ('in_state', 'documented_by', 'multisource_inlined_rel'): + assert execute('SET S cw_support RT WHERE RT name %(rtype)s, RT is CWRType, S eid %(s)s', + {'rtype': rtype, 's': source.eid}) + + def setup_module(*args): global repo2, cnx2, repo3, cnx3 cfg1 = ExternalSource1Configuration('data', apphome=TwoSourcesTC.datadir) repo2, cnx2 = init_test_database(config=cfg1) cfg2 = ExternalSource2Configuration('data', apphome=TwoSourcesTC.datadir) repo3, cnx3 = init_test_database(config=cfg2) - cnx3.request().create_entity('CWSource', name=u'extern', type=u'pyrorql', - config=EXTERN_SOURCE_CFG) + src = cnx3.request().create_entity('CWSource', name=u'extern', + type=u'pyrorql', config=EXTERN_SOURCE_CFG) + add_extern_mapping(src) cnx3.commit() TestServerConfiguration.no_sqlite_wrap = True @@ -106,11 +116,11 @@ pyro-ns-id = extern-multi cubicweb-user = admin cubicweb-password = gingkow -mapping-file = extern_mapping.py ''')]: - self.request().create_entity('CWSource', name=unicode(uri), - type=u'pyrorql', - config=unicode(config)) + source = self.request().create_entity( + 'CWSource', name=unicode(uri), type=u'pyrorql', + config=unicode(config)) + add_extern_mapping(source) self.commit() # trigger discovery self.sexecute('Card X') diff -r a2ccbcbb08a6 -r 24bf6f181d0e web/views/cwsources.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/web/views/cwsources.py Wed Dec 01 17:11:35 2010 +0100 @@ -0,0 +1,171 @@ +# copyright 2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved. +# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr +# +# This file is part of CubicWeb. +# +# CubicWeb is free software: you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation, either version 2.1 of the License, or (at your option) +# any later version. +# +# CubicWeb is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License along +# with CubicWeb. If not, see . +"""Specific views for data sources""" + +__docformat__ = "restructuredtext en" +_ = unicode + +from itertools import repeat, chain + +from cubicweb.selectors import is_instance, score_entity +from cubicweb.view import EntityView +from cubicweb.schema import META_RTYPES, VIRTUAL_RTYPES, display_name +from cubicweb.web import uicfg +from cubicweb.web.views import tabs + +for rtype in ('cw_support', 'cw_may_cross', 'cw_dont_cross'): + uicfg.primaryview_section.tag_subject_of(('CWSource', rtype, '*'), + 'hidden') + +class CWSourcePrimaryView(tabs.TabbedPrimaryView): + __select__ = is_instance('CWSource') + tabs = [_('cwsource-main'), _('cwsource-mapping')] + default_tab = 'cwsource-main' + + +class CWSourceMainTab(tabs.PrimaryTab): + __regid__ = 'cwsource-main' + __select__ = tabs.PrimaryTab.__select__ & is_instance('CWSource') + + +class CWSourceMappingTab(EntityView): + __regid__ = 'cwsource-mapping' + __select__ = (tabs.PrimaryTab.__select__ & is_instance('CWSource') + & score_entity(lambda x:x.type == 'pyrorql')) + + def entity_call(self, entity): + _ = self._cw._ + self.w('

%s

' % _('Entity and relation types supported by this source')) + self.wview('list', entity.related('cw_support'), 'noresult') + self.w('

%s

' % _('Relations that should not be crossed')) + self.w('

%s

' % _( + 'By default, when a relation is not supported by a source, it is ' + 'supposed that a local relation may point to an entity from the ' + 'external source. Relations listed here won\'t have this ' + '"crossing" behaviour.')) + self.wview('list', entity.related('cw_dont_cross'), 'noresult') + self.w('

%s

' % _('Relations that can be crossed')) + self.w('

%s

' % _( + 'By default, when a relation is supported by a source, it is ' + 'supposed that a local relation can\'t point to an entity from the ' + 'external source. Relations listed here may have this ' + '"crossing" behaviour anyway.')) + self.wview('list', entity.related('cw_may_cross'), 'noresult') + if self._cw.user.is_in_group('managers'): + errors, warnings, infos = check_mapping(entity) + if (errors or warnings or infos): + self.w('

%s

' % _('Detected problems')) + errors = zip(repeat(_('error'), errors)) + warnings = zip(repeat(_('warning'), warnings)) + infos = zip(repeat(_('warning'), infos)) + self.wview('pyvaltable', pyvalue=chain(errors, warnings, infos)) + +def check_mapping(cwsource): + req = cwsource._cw + _ = req._ + errors = [] + error = errors.append + warnings = [] + warning = warnings.append + infos = [] + info = infos.append + srelations = set() + sentities = set() + maycross = set() + dontcross = set() + # first check supported stuff / meta & virtual types and get mapping as sets + for cwertype in cwsource.cw_support: + if cwertype.name in META_RTYPES: + error(_('meta relation %s can not be supported') % cwertype.name) + else: + if cwertype.__regid__ == 'CWEType': + sentities.add(cwertype.name) + else: + srelations.add(cwertype.name) + for attr, attrset in (('cw_may_cross', maycross), + ('cw_dont_cross', dontcross)): + for cwrtype in getattr(cwsource, attr): + if cwrtype.name in VIRTUAL_RTYPES: + error(_('virtual relation %(rtype)s can not be referenced by ' + 'the "%(srel)s" relation') % + {'rtype': cwrtype.name, + 'srel': display_name(req, attr, context='CWSource')}) + else: + attrset.add(cwrtype.name) + # check relation in dont_cross_relations aren't in support_relations + for rtype in dontcross & maycross: + info(_('relation %(rtype)s is supported but in %(dontcross)s') % + {'rtype': rtype, + 'dontcross': display_name(req, 'cw_dont_cross', + context='CWSource')}) + # check relation in cross_relations are in support_relations + for rtype in maycross & srelations: + info(_('relation %(rtype)s isn\'t supported but in %(maycross)s') % + {'rtype': rtype, + 'dontcross': display_name(req, 'cw_may_cross', + context='CWSource')}) + # now check for more handy things + seen = set() + for etype in sentities: + eschema = req.vreg.schema[etype] + for rschema, ttypes, role in eschema.relation_definitions(): + if rschema in META_RTYPES: + continue + ttypes = [ttype for ttype in ttypes if ttype in sentities] + if not rschema in srelations: + somethingprinted = False + for ttype in ttypes: + rdef = rschema.role_rdef(etype, ttype, role) + seen.add(rdef) + if rdef.role_cardinality(role) in '1+': + error(_('relation %(type)s with %(etype)s as %(role)s ' + 'and target type %(target)s is mandatory but ' + 'not supported') % + {'rtype': rschema, 'etype': etype, 'role': role, + 'target': ttype}) + somethingprinted = True + elif ttype in sentities: + if rdef not in seen: + warning(_('%s could be supported') % rdef) + somethingprinted = True + if rschema not in dontcross: + if role == 'subject' and rschema.inlined: + error(_('inlined relation %(rtype)s of %(etype)s ' + 'should be supported') % + {'rtype': rschema, 'etype': etype}) + elif (not somethingprinted and rschema not in seen + and rschema not in maycross): + info(_('you may want to specify something for %s') % + rschema) + seen.add(rschema) + else: + if not ttypes: + warning(_('relation %(rtype)s with %(etype)s as %(role)s ' + 'is supported but no target type supported') % + {'rtype': rschema, 'role': role, 'etype': etype}) + if rschema in maycross and rschema.inlined: + error(_('you should un-inline relation %s which is ' + 'supported and may be crossed ') % rschema) + for rschema in srelations: + for subj, obj in rschema.rdefs: + if subj in sentities and obj in sentities: + break + else: + error(_('relation %s is supported but none if its definitions ' + 'matches supported entities') % rschema) + return errors, warnings, infos