[cwvreg] load registry using modules names instead of directories
Introspect cubicweb, cubes and apphome using pkgutil to generate the full list
of modules names for loading registries.
Avoiding using bogus logilab.common.modutils.modpath_from_file().
--- a/cubicweb/cwconfig.py Thu Jan 19 15:27:39 2017 +0100
+++ b/cubicweb/cwconfig.py Thu Jan 19 14:53:05 2017 +0100
@@ -823,57 +823,12 @@
modnames.append(('data', modname))
return modnames
- def appobjects_path(self):
- """return a list of files or directories where the registry will look
- for application objects. By default return nothing in NoApp config.
+ def appobjects_modnames(self):
+ """return a list of modules where the registry will look for
+ application objects. By default return nothing in NoApp config.
"""
return []
- def build_appobjects_path(self, templpath, evobjpath=None, tvobjpath=None):
- """given a list of directories, return a list of sub files and
- directories that should be loaded by the instance objects registry.
-
- :param evobjpath:
- optional list of sub-directories (or files without the .py ext) of
- the cubicweb library that should be tested and added to the output list
- if they exists. If not give, default to `cubicweb_appobject_path` class
- attribute.
- :param tvobjpath:
- optional list of sub-directories (or files without the .py ext) of
- directories given in `templpath` that should be tested and added to
- the output list if they exists. If not give, default to
- `cube_appobject_path` class attribute.
- """
- vregpath = self.build_appobjects_cubicweb_path(evobjpath)
- vregpath += self.build_appobjects_cube_path(templpath, tvobjpath)
- return vregpath
-
- def build_appobjects_cubicweb_path(self, evobjpath=None):
- vregpath = []
- if evobjpath is None:
- evobjpath = self.cubicweb_appobject_path
- # NOTE: for the order, see http://www.cubicweb.org/ticket/2330799
- # it is clearly a workaround
- for subdir in sorted(evobjpath, key=lambda x:x != 'entities'):
- path = join(CW_SOFTWARE_ROOT, subdir)
- if exists(path):
- vregpath.append(path)
- return vregpath
-
- def build_appobjects_cube_path(self, templpath, tvobjpath=None):
- vregpath = []
- if tvobjpath is None:
- tvobjpath = self.cube_appobject_path
- for directory in templpath:
- # NOTE: for the order, see http://www.cubicweb.org/ticket/2330799
- for subdir in sorted(tvobjpath, key=lambda x:x != 'entities'):
- path = join(directory, subdir)
- if exists(path):
- vregpath.append(path)
- elif exists(path + '.py'):
- vregpath.append(path + '.py')
- return vregpath
-
apphome = None
def load_site_cubicweb(self, cubes=()):
@@ -1361,14 +1316,42 @@
self.exception('localisation support error for language %s',
language)
- def appobjects_path(self):
- """return a list of files or directories where the registry will look
- for application objects
- """
- templpath = list(reversed(self.cubes_path()))
- if self.apphome: # may be unset in tests
- templpath.append(self.apphome)
- return self.build_appobjects_path(templpath)
+ @staticmethod
+ def _sorted_appobjects(appobjects):
+ appobjects = sorted(appobjects)
+ try:
+ index = appobjects.index('entities')
+ except ValueError:
+ pass
+ else:
+ # put entities first
+ appobjects.insert(0, appobjects.pop(index))
+ return appobjects
+
+ def _appobjects_cube_modnames(self, cube):
+ modnames = []
+ cube_submodnames = self._sorted_appobjects(self.cube_appobject_path)
+ for name in cube_submodnames:
+ for modname, filepath in _expand_modname('.'.join(['cubes', cube, name])):
+ modnames.append(modname)
+ return modnames
+
+ def appobjects_modnames(self):
+ modnames = []
+ for name in self._sorted_appobjects(self.cubicweb_appobject_path):
+ for modname, filepath in _expand_modname('cubicweb.' + name):
+ modnames.append(modname)
+ for cube in reversed(self.cubes()):
+ modnames.extend(self._appobjects_cube_modnames(cube))
+ if self.apphome:
+ cube_submodnames = self._sorted_appobjects(self.cube_appobject_path)
+ apphome = realpath(self.apphome)
+ for name in cube_submodnames:
+ for modname, filepath in _expand_modname(name):
+ # ensure file is in apphome
+ if realpath(filepath).startswith(apphome):
+ modnames.append(modname)
+ return modnames
def set_sources_mode(self, sources):
if not 'all' in sources:
--- a/cubicweb/cwvreg.py Thu Jan 19 15:27:39 2017 +0100
+++ b/cubicweb/cwvreg.py Thu Jan 19 14:53:05 2017 +0100
@@ -29,7 +29,7 @@
from logilab.common.decorators import cached, clear_cache
from logilab.common.deprecation import class_deprecated
-from logilab.common.modutils import cleanup_sys_modules
+from logilab.common.modutils import clean_sys_modules
from logilab.common.registry import RegistryStore, Registry, ObjectNotFound, RegistryNotFound
from rql import RQLHelper
@@ -417,7 +417,7 @@
"""set instance'schema and load application objects"""
self._set_schema(schema)
# now we can load application's web objects
- self.reload(self.config.appobjects_path(), force_reload=False)
+ self.reload(self.config.appobjects_modnames(), force_reload=False)
# map lowered entity type names to their actual name
self.case_insensitive_etypes = {}
for eschema in self.schema.entities():
@@ -426,13 +426,28 @@
clear_cache(eschema, 'ordered_relations')
clear_cache(eschema, 'meta_attributes')
+ def is_reload_needed(self, modnames):
+ """overriden to handle modules names instead of directories"""
+ lastmodifs = self._lastmodifs
+ for modname in modnames:
+ if modname not in sys.modules:
+ # new module to load
+ return True
+ filepath = sys.modules[modname].__file__
+ if filepath.endswith('.py'):
+ mdate = self._mdate(filepath)
+ if filepath not in lastmodifs or lastmodifs[filepath] < mdate:
+ self.info('File %s changed since last visit', filepath)
+ return True
+ return False
+
def reload_if_needed(self):
- path = self.config.appobjects_path()
- if self.is_reload_needed(path):
- self.reload(path)
+ modnames = self.config.appobjects_modnames()
+ if self.is_reload_needed(modnames):
+ self.reload(modnames)
- def _cleanup_sys_modules(self, path):
- """Remove submodules of `directories` from `sys.modules` and cleanup
+ def _cleanup_sys_modules(self, modnames):
+ """Remove modules and submodules of `modnames` from `sys.modules` and cleanup
CW_EVENT_MANAGER accordingly.
We take care to properly remove obsolete registry callbacks.
@@ -446,18 +461,18 @@
# for non-function callable, we do nothing interesting
module = getattr(func, '__module__', None)
caches[id(callback)] = module
- deleted_modules = set(cleanup_sys_modules(path))
+ deleted_modules = set(clean_sys_modules(modnames))
for callbacklist in callbackdata:
for callback in callbacklist[:]:
module = caches[id(callback)]
if module and module in deleted_modules:
callbacklist.remove(callback)
- def reload(self, path, force_reload=True):
+ def reload(self, modnames, force_reload=True):
"""modification detected, reset and reload the vreg"""
CW_EVENT_MANAGER.emit('before-registry-reload')
if force_reload:
- self._cleanup_sys_modules(path)
+ self._cleanup_sys_modules(modnames)
cubes = self.config.cubes()
# if the fs code use some cubes not yet registered into the instance
# we should cleanup sys.modules for those as well to avoid potential
@@ -465,9 +480,9 @@
cfg = self.config
for cube in cfg.expand_cubes(cubes, with_recommends=True):
if not cube in cubes:
- cpath = cfg.build_appobjects_cube_path([cfg.cube_dir(cube)])
- self._cleanup_sys_modules(cpath)
- self.register_objects(path)
+ cube_modnames = cfg.appobjects_cube_modnames(cube)
+ self._cleanup_sys_modules(cube_modnames)
+ self.register_modnames(modnames)
CW_EVENT_MANAGER.emit('after-registry-reload')
def load_file(self, filepath, modname):
--- a/cubicweb/devtools/devctl.py Thu Jan 19 15:27:39 2017 +0100
+++ b/cubicweb/devtools/devctl.py Thu Jan 19 14:53:05 2017 +0100
@@ -37,6 +37,7 @@
from six.moves import input
from logilab.common import STD_BLACKLIST
+from logilab.common.modutils import clean_sys_modules
from logilab.common.fileutils import ensure_fs_mode
from logilab.common.shellutils import find
@@ -100,24 +101,6 @@
return None
-def cleanup_sys_modules(config):
- # cleanup sys.modules, required when we're updating multiple cubes
- appobjects_path = config.appobjects_path()
- for name, mod in list(sys.modules.items()):
- if mod is None:
- # duh ? logilab.common.os for instance
- del sys.modules[name]
- continue
- if not hasattr(mod, '__file__'):
- continue
- if mod.__file__ is None:
- # odd/rare but real
- continue
- for path in appobjects_path:
- if mod.__file__.startswith(path):
- del sys.modules[name]
- break
-
def generate_schema_pot(w, cubedir=None):
"""generate a pot file with schema specific i18n messages
@@ -136,7 +119,7 @@
else:
config = DevConfiguration()
cube = libconfig = None
- cleanup_sys_modules(config)
+ clean_sys_modules(config.appobjects_modnames())
schema = config.load_schema(remove_unused_rtypes=False)
vreg = CWRegistryStore(config)
# set_schema triggers objects registrations
@@ -161,7 +144,7 @@
# (cubicweb incl.)
from cubicweb.cwvreg import CWRegistryStore
libschema = libconfig.load_schema(remove_unused_rtypes=False)
- cleanup_sys_modules(libconfig)
+ clean_sys_modules(libconfig.appobjects_modnames())
libvreg = CWRegistryStore(libconfig)
libvreg.set_schema(libschema) # trigger objects registration
libafss = libvreg['uicfg']['autoform_section']
--- a/cubicweb/test/unittest_cwconfig.py Thu Jan 19 15:27:39 2017 +0100
+++ b/cubicweb/test/unittest_cwconfig.py Thu Jan 19 14:53:05 2017 +0100
@@ -178,17 +178,6 @@
self.assertEqual(self.config.expand_cubes(('email', 'comment')),
['email', 'comment', 'file'])
- def test_appobjects_path(self):
- path = [unabsolutize(p) for p in self.config.appobjects_path()]
- self.assertEqual(path[0], 'entities')
- self.assertCountEqual(path[1:4], ['web/views', 'sobjects', 'hooks'])
- self.assertEqual(path[4], 'file/entities')
- self.assertCountEqual(path[5:7],
- ['file/views.py', 'file/hooks'])
- self.assertEqual(path[7], 'email/entities.py')
- self.assertCountEqual(path[8:10],
- ['email/views', 'email/hooks.py'])
- self.assertEqual(path[10:], ['test/data/entities.py', 'test/data/views.py'])
def test_init_cubes_ignore_pyramid_cube(self):
warning_msg = 'cubicweb-pyramid got integrated into CubicWeb'
@@ -463,7 +452,48 @@
join(libdir, 'schema.py'))
self.assertEqual(config.schema_modnames(), expected)
-
+ @templibdir
+ def test_appobjects_modnames(self, libdir):
+ for filepath in (
+ join(libdir, 'entities.py'),
+ join(libdir, 'cubicweb_foo', '__init__.py'),
+ join(libdir, 'cubicweb_foo', 'entities', '__init__.py'),
+ join(libdir, 'cubicweb_foo', 'entities', 'a.py'),
+ join(libdir, 'cubicweb_foo', 'hooks.py'),
+ join(libdir, 'cubes', '__init__.py'),
+ join(libdir, 'cubes', 'bar', '__init__.py'),
+ join(libdir, 'cubes', 'bar', 'hooks.py'),
+ join(libdir, '_instance_dir', 'data1', 'entities.py'),
+ join(libdir, '_instance_dir', 'data2', 'hooks.py'),
+ ):
+ create_filepath(filepath)
+ instance_dir, cubes_dir = (
+ join(libdir, '_instance_dir'), join(libdir, 'cubes'))
+ expected = [
+ 'cubicweb.entities',
+ 'cubicweb.entities.adapters',
+ 'cubicweb.entities.authobjs',
+ 'cubicweb.entities.lib',
+ 'cubicweb.entities.schemaobjs',
+ 'cubicweb.entities.sources',
+ 'cubicweb.entities.wfobjs',
+ 'cubes.bar.hooks',
+ 'cubes.foo.entities',
+ 'cubes.foo.entities.a',
+ 'cubes.foo.hooks',
+ ]
+ # data1 has entities
+ with temp_config('data1', instance_dir, cubes_dir,
+ ('foo', 'bar')) as config:
+ config.cube_appobject_path = set(['entities', 'hooks'])
+ self.assertEqual(config.appobjects_modnames(),
+ expected + ['entities'])
+ # data2 has hooks
+ with temp_config('data2', instance_dir, cubes_dir,
+ ('foo', 'bar')) as config:
+ config.cube_appobject_path = set(['entities', 'hooks'])
+ self.assertEqual(config.appobjects_modnames(),
+ expected + ['hooks'])
if __name__ == '__main__':
--- a/cubicweb/web/webconfig.py Thu Jan 19 15:27:39 2017 +0100
+++ b/cubicweb/web/webconfig.py Thu Jan 19 14:53:05 2017 +0100
@@ -82,7 +82,7 @@
"""the WebConfiguration is a singleton object handling instance's
configuration and preferences
"""
- cubicweb_appobject_path = CubicWebConfiguration.cubicweb_appobject_path | set([join('web', 'views')])
+ cubicweb_appobject_path = CubicWebConfiguration.cubicweb_appobject_path | set(['web.views'])
cube_appobject_path = CubicWebConfiguration.cube_appobject_path | set(['views'])
options = merge_options(CubicWebConfiguration.options + (