# HG changeset patch # User Philippe Pepiot # Date 1484836059 -3600 # Node ID bf6106b9163316ec251e57fa329510cb417e2fdc # Parent c5d3382f14e9e5153f0725c850f28e9cdb470bf3 [schema] load schema from modules names instead of directories Introspect cubicweb, cubes and apphome using pkgutil to generate the full list of modules names for loading the schema. Keep historical behavior and check if source .py file exists if a module is found using python bytecode file (.pyc and .pyo) Loading schema from apphome require apphome to be present in sys.path and that "schema" module resolve to a file located in apphome. Update migraction tests to explicitely update sys.path when loading schema from different apps, use a contextmanager for this so it's more readable. Require updated logilab-common and yams diff -r c5d3382f14e9 -r bf6106b91633 cubicweb/cwconfig.py --- a/cubicweb/cwconfig.py Wed Jan 18 17:16:00 2017 +0100 +++ b/cubicweb/cwconfig.py Thu Jan 19 15:27:39 2017 +0100 @@ -185,7 +185,7 @@ import logging.config import os from os.path import (exists, join, expanduser, abspath, normpath, - basename, isdir, dirname, splitext) + basename, isdir, dirname, splitext, realpath) import pkgutil import pkg_resources import re @@ -275,6 +275,40 @@ return cube +def _expand_modname(modname): + """expand modules names `modname` if exists by walking non package submodules + and yield (submodname, filepath) including `modname` itself + + If the file ends with .pyc or .pyo (python bytecode) also check that the + corresponding source .py file exists before yielding. + """ + try: + loader = pkgutil.find_loader(modname) + except ImportError: + return + if not loader: + return + + def check_source_file(filepath): + if filepath[-4:] in ('.pyc', '.pyo'): + if not exists(filepath[:-1]): + return False + return True + + filepath = loader.get_filename() + if not check_source_file(filepath): + return + yield modname, filepath + if loader.is_package(modname): + path = dirname(filepath) + for subloader, subname, ispkg in pkgutil.walk_packages([path]): + # ignore subpackages (historical behavior) + if not ispkg: + filepath = subloader.find_module(subname).get_filename() + if check_source_file(filepath): + yield modname + '.' + subname, filepath + + # persistent options definition PERSISTENT_OPTIONS = ( ('encoding', @@ -775,6 +809,20 @@ # configure simpleTal logger logging.getLogger('simpleTAL').setLevel(logging.ERROR) + def schema_modnames(self): + modnames = [] + for name in ('bootstrap', 'base', 'workflow', 'Bookmark'): + modnames.append(('cubicweb', 'cubicweb.schemas.' + name)) + for cube in reversed(self.cubes()): + for modname, filepath in _expand_modname('cubes.{0}.schema'.format(cube)): + modnames.append((cube, modname)) + if self.apphome: + apphome = realpath(self.apphome) + for modname, filepath in _expand_modname('schema'): + if realpath(filepath).startswith(apphome): + modnames.append(('data', modname)) + return modnames + def appobjects_path(self): """return a list of files or directories where the registry will look for application objects. By default return nothing in NoApp config. diff -r c5d3382f14e9 -r bf6106b91633 cubicweb/schema.py --- a/cubicweb/schema.py Wed Jan 18 17:16:00 2017 +0100 +++ b/cubicweb/schema.py Thu Jan 19 15:27:39 2017 +0100 @@ -19,6 +19,7 @@ from __future__ import print_function +import pkgutil import re from os.path import join, basename from hashlib import md5 @@ -1369,19 +1370,12 @@ """ schemacls = CubicWebSchema - def load(self, config, path=(), **kwargs): + def load(self, config, modnames=(['cubicweb', 'cubicweb.schemas.bootstrap'],), **kwargs): """return a Schema instance from the schema definition read from """ return super(BootstrapSchemaLoader, self).load( - path, config.appid, register_base_types=False, **kwargs) - - def _load_definition_files(self, cubes=None): - # bootstraping, ignore cubes - filepath = join(cubicweb.CW_SOFTWARE_ROOT, 'schemas', 'bootstrap.py') - self.info('loading %s', filepath) - with tempattr(ybo, 'PACKAGE', 'cubicweb'): # though we don't care here - self.handle_file(filepath) + modnames, name=config.appid, register_base_types=False, **kwargs) def unhandled_file(self, filepath): """called when a file without handler associated has been found""" @@ -1402,30 +1396,12 @@ from """ self.info('loading %s schemas', ', '.join(config.cubes())) - self.extrapath = config.extrapath - if config.apphome: - path = tuple(reversed([config.apphome] + config.cubes_path())) - else: - path = tuple(reversed(config.cubes_path())) try: - return super(CubicWebSchemaLoader, self).load(config, path=path, **kwargs) + return super(CubicWebSchemaLoader, self).load(config, config.schema_modnames(), **kwargs) finally: # we've to cleanup modules imported from cubicweb.schemas as well cleanup_sys_modules([join(cubicweb.CW_SOFTWARE_ROOT, 'schemas')]) - def _load_definition_files(self, cubes): - for filepath in (join(cubicweb.CW_SOFTWARE_ROOT, 'schemas', 'bootstrap.py'), - join(cubicweb.CW_SOFTWARE_ROOT, 'schemas', 'base.py'), - join(cubicweb.CW_SOFTWARE_ROOT, 'schemas', 'workflow.py'), - join(cubicweb.CW_SOFTWARE_ROOT, 'schemas', 'Bookmark.py')): - self.info('loading %s', filepath) - with tempattr(ybo, 'PACKAGE', 'cubicweb'): - self.handle_file(filepath) - for cube in cubes: - for filepath in self.get_schema_files(cube): - with tempattr(ybo, 'PACKAGE', basename(cube)): - self.handle_file(filepath) - # these are overridden by set_log_methods below # only defining here to prevent pylint from complaining info = warning = error = critical = exception = debug = lambda msg, *a, **kw: None diff -r c5d3382f14e9 -r bf6106b91633 cubicweb/server/test/unittest_migractions.py --- a/cubicweb/server/test/unittest_migractions.py Wed Jan 18 17:16:00 2017 +0100 +++ b/cubicweb/server/test/unittest_migractions.py Thu Jan 19 15:27:39 2017 +0100 @@ -19,6 +19,7 @@ import os import os.path as osp +import sys from datetime import date from contextlib import contextmanager import tempfile @@ -77,13 +78,25 @@ # we have to read schema from the database to get eid for schema entities self.repo.set_schema(self.repo.deserialize_schema(), resetvreg=False) # hack to read the schema from data/migrschema - config = self.config - config.appid = osp.join(self.appid, 'migratedapp') - config._apphome = osp.join(HERE, config.appid) - global migrschema - migrschema = config.load_schema() - config.appid = self.appid - config._apphome = osp.join(HERE, self.appid) + + @contextmanager + def temp_app(config, appid, apphome): + old = config.apphome, config.appid + sys.path.remove(old[0]) + sys.path.insert(0, apphome) + config._apphome, config.appid = apphome, appid + try: + yield config + finally: + sys.path.remove(apphome) + sys.path.insert(0, old[0]) + config._apphome, config.appid = old + + appid = osp.join(self.appid, 'migratedapp') + apphome = osp.join(HERE, appid) + with temp_app(self.config, appid, apphome) as config: + global migrschema + migrschema = config.load_schema() def setUp(self): self.configcls.cls_adjust_sys_path() diff -r c5d3382f14e9 -r bf6106b91633 cubicweb/test/unittest_cwconfig.py --- a/cubicweb/test/unittest_cwconfig.py Wed Jan 18 17:16:00 2017 +0100 +++ b/cubicweb/test/unittest_cwconfig.py Thu Jan 19 15:27:39 2017 +0100 @@ -17,8 +17,12 @@ # with CubicWeb. If not, see . """cubicweb.cwconfig unit tests""" +import contextlib +import compileall +import functools import sys import os +import pkgutil from os.path import dirname, join, abspath from pkg_resources import EntryPoint, Distribution import unittest @@ -31,7 +35,8 @@ from cubicweb.devtools import ApptestConfiguration from cubicweb.devtools.testlib import BaseTestCase, TemporaryDirectory -from cubicweb.cwconfig import _find_prefix +from cubicweb.cwconfig import ( + CubicWebConfiguration, _find_prefix, _expand_modname) def unabsolutize(path): @@ -44,6 +49,50 @@ raise Exception('duh? %s' % path) +def templibdir(func): + """create a temporary directory and insert it in sys.path""" + @functools.wraps(func) + def wrapper(*args, **kwargs): + with TemporaryDirectory() as libdir: + sys.path.insert(0, libdir) + try: + args = args + (libdir,) + return func(*args, **kwargs) + finally: + sys.path.remove(libdir) + return wrapper + + +def create_filepath(filepath): + filedir = dirname(filepath) + if not os.path.exists(filedir): + os.makedirs(filedir) + with open(filepath, 'a'): + pass + + +@contextlib.contextmanager +def temp_config(appid, instance_dir, cubes_dir, cubes): + """context manager that create a config object with specified appid, + instance_dir, cubes_dir and cubes""" + cls = CubicWebConfiguration + old = (cls._INSTANCES_DIR, cls.CUBES_DIR, cls.CUBES_PATH, + sys.path[:], sys.meta_path[:]) + old_modules = set(sys.modules) + try: + cls._INSTANCES_DIR, cls.CUBES_DIR, cls.CUBES_PATH = ( + instance_dir, cubes_dir, []) + config = cls(appid) + config._cubes = cubes + config.adjust_sys_path() + yield config + finally: + (cls._INSTANCES_DIR, cls.CUBES_DIR, cls.CUBES_PATH, + sys.path[:], sys.meta_path[:]) = old + for module in set(sys.modules) - old_modules: + del sys.modules[module] + + class CubicWebConfigurationTC(BaseTestCase): @classmethod @@ -313,5 +362,109 @@ os.environ['VIRTUAL_ENV'] = venv +class ModnamesTC(unittest.TestCase): + + @templibdir + def test_expand_modnames(self, libdir): + tempdir = join(libdir, 'lib') + filepaths = [ + join(tempdir, '__init__.py'), + join(tempdir, 'a.py'), + join(tempdir, 'b.py'), + join(tempdir, 'c.py'), + join(tempdir, 'b', '__init__.py'), + join(tempdir, 'b', 'a.py'), + join(tempdir, 'b', 'c.py'), + join(tempdir, 'b', 'd', '__init__.py'), + join(tempdir, 'e', 'e.py'), + ] + for filepath in filepaths: + create_filepath(filepath) + # not importable + self.assertEqual(list(_expand_modname('isnotimportable')), []) + # not a python package + self.assertEqual(list(_expand_modname('lib.e')), []) + self.assertEqual(list(_expand_modname('lib.a')), [ + ('lib.a', join(tempdir, 'a.py')), + ]) + # lib.b.d (subpackage) not to be imported + self.assertEqual(list(_expand_modname('lib.b')), [ + ('lib.b', join(tempdir, 'b', '__init__.py')), + ('lib.b.a', join(tempdir, 'b', 'a.py')), + ('lib.b.c', join(tempdir, 'b', 'c.py')), + ]) + self.assertEqual(list(_expand_modname('lib')), [ + ('lib', join(tempdir, '__init__.py')), + ('lib.a', join(tempdir, 'a.py')), + ('lib.c', join(tempdir, 'c.py')), + ]) + for source in ( + join(tempdir, 'c.py'), + join(tempdir, 'b', 'c.py'), + ): + if not PY3: + # ensure pyc file exists. + # Doesn't required for PY3 since it create __pycache__ + # directory and will not import if source file doesn't + # exists. + compileall.compile_file(source, force=True) + self.assertTrue(os.path.exists(source + 'c')) + # remove source file + os.remove(source) + self.assertEqual(list(_expand_modname('lib.c')), []) + self.assertEqual(list(_expand_modname('lib.b')), [ + ('lib.b', join(tempdir, 'b', '__init__.py')), + ('lib.b.a', join(tempdir, 'b', 'a.py')), + ]) + self.assertEqual(list(_expand_modname('lib')), [ + ('lib', join(tempdir, '__init__.py')), + ('lib.a', join(tempdir, 'a.py')), + ]) + + @templibdir + def test_schema_modnames(self, libdir): + for filepath in ( + join(libdir, 'schema.py'), + join(libdir, 'cubicweb_foo', '__init__.py'), + join(libdir, 'cubicweb_foo', 'schema', '__init__.py'), + join(libdir, 'cubicweb_foo', 'schema', 'a.py'), + join(libdir, 'cubicweb_foo', 'schema', 'b.py'), + join(libdir, 'cubes', '__init__.py'), + join(libdir, 'cubes', 'bar', '__init__.py'), + join(libdir, 'cubes', 'bar', 'schema.py'), + join(libdir, '_instance_dir', 'data1', 'schema.py'), + join(libdir, '_instance_dir', 'data2', 'noschema.py'), + ): + create_filepath(filepath) + expected = [ + ('cubicweb', 'cubicweb.schemas.bootstrap'), + ('cubicweb', 'cubicweb.schemas.base'), + ('cubicweb', 'cubicweb.schemas.workflow'), + ('cubicweb', 'cubicweb.schemas.Bookmark'), + ('bar', 'cubes.bar.schema'), + ('foo', 'cubes.foo.schema'), + ('foo', 'cubes.foo.schema.a'), + ('foo', 'cubes.foo.schema.b'), + ] + # app has schema file + instance_dir, cubes_dir = ( + join(libdir, '_instance_dir'), join(libdir, 'cubes')) + with temp_config('data1', instance_dir, cubes_dir, + ('foo', 'bar')) as config: + self.assertEqual(pkgutil.find_loader('schema').get_filename(), + join(libdir, '_instance_dir', + 'data1', 'schema.py')) + self.assertEqual(config.schema_modnames(), + expected + [('data', 'schema')]) + # app doesn't have schema file + with temp_config('data2', instance_dir, cubes_dir, + ('foo', 'bar')) as config: + self.assertEqual(pkgutil.find_loader('schema').get_filename(), + join(libdir, 'schema.py')) + self.assertEqual(config.schema_modnames(), expected) + + + + if __name__ == '__main__': unittest.main() diff -r c5d3382f14e9 -r bf6106b91633 cubicweb/test/unittest_schema.py --- a/cubicweb/test/unittest_schema.py Wed Jan 18 17:16:00 2017 +0100 +++ b/cubicweb/test/unittest_schema.py Thu Jan 19 15:27:39 2017 +0100 @@ -402,7 +402,7 @@ self.loader.post_build_callbacks = [] def _test(self, schemafile, msg): - self.loader.handle_file(join(DATADIR, schemafile)) + self.loader.handle_file(join(DATADIR, schemafile), None) sch = self.loader.schemacls('toto') with self.assertRaises(BadSchemaDefinition) as cm: fill_schema(sch, self.loader.defined, False) diff -r c5d3382f14e9 -r bf6106b91633 requirements/dev.txt --- a/requirements/dev.txt Wed Jan 18 17:16:00 2017 +0100 +++ b/requirements/dev.txt Thu Jan 19 15:27:39 2017 +0100 @@ -1,1 +1,3 @@ pytest +http://hg.logilab.org/master/logilab/common/archive/default.tar.bz2#egg=logilab-common +http://hg.logilab.org/master/yams/archive/default.tar.bz2#egg=yams