# HG changeset patch
# User Philippe Pepiot <philippe.pepiot@logilab.fr>
# Date 1484833985 -3600
# Node ID 8496135b6dc16dca0799e57c1eab5843a50b8509
# Parent  bf6106b9163316ec251e57fa329510cb417e2fdc
[cwvreg] load registry using modules names instead of directories

Introspect cubicweb, cubes and apphome using pkgutil to generate the full list
of modules names for loading registries.
Avoiding using bogus logilab.common.modutils.modpath_from_file().

diff -r bf6106b91633 -r 8496135b6dc1 cubicweb/cwconfig.py
--- a/cubicweb/cwconfig.py	Thu Jan 19 15:27:39 2017 +0100
+++ b/cubicweb/cwconfig.py	Thu Jan 19 14:53:05 2017 +0100
@@ -823,57 +823,12 @@
                     modnames.append(('data', modname))
         return modnames
 
-    def appobjects_path(self):
-        """return a list of files or directories where the registry will look
-        for application objects. By default return nothing in NoApp config.
+    def appobjects_modnames(self):
+        """return a list of modules where the registry will look for
+        application objects. By default return nothing in NoApp config.
         """
         return []
 
-    def build_appobjects_path(self, templpath, evobjpath=None, tvobjpath=None):
-        """given a list of directories, return a list of sub files and
-        directories that should be loaded by the instance objects registry.
-
-        :param evobjpath:
-          optional list of sub-directories (or files without the .py ext) of
-          the cubicweb library that should be tested and added to the output list
-          if they exists. If not give, default to `cubicweb_appobject_path` class
-          attribute.
-        :param tvobjpath:
-          optional list of sub-directories (or files without the .py ext) of
-          directories given in `templpath` that should be tested and added to
-          the output list if they exists. If not give, default to
-          `cube_appobject_path` class attribute.
-        """
-        vregpath = self.build_appobjects_cubicweb_path(evobjpath)
-        vregpath += self.build_appobjects_cube_path(templpath, tvobjpath)
-        return vregpath
-
-    def build_appobjects_cubicweb_path(self, evobjpath=None):
-        vregpath = []
-        if evobjpath is None:
-            evobjpath = self.cubicweb_appobject_path
-        # NOTE: for the order, see http://www.cubicweb.org/ticket/2330799
-        #       it is clearly a workaround
-        for subdir in sorted(evobjpath, key=lambda x:x != 'entities'):
-            path = join(CW_SOFTWARE_ROOT, subdir)
-            if exists(path):
-                vregpath.append(path)
-        return vregpath
-
-    def build_appobjects_cube_path(self, templpath, tvobjpath=None):
-        vregpath = []
-        if tvobjpath is None:
-            tvobjpath = self.cube_appobject_path
-        for directory in templpath:
-            # NOTE: for the order, see http://www.cubicweb.org/ticket/2330799
-            for subdir in sorted(tvobjpath, key=lambda x:x != 'entities'):
-                path = join(directory, subdir)
-                if exists(path):
-                    vregpath.append(path)
-                elif exists(path + '.py'):
-                    vregpath.append(path + '.py')
-        return vregpath
-
     apphome = None
 
     def load_site_cubicweb(self, cubes=()):
@@ -1361,14 +1316,42 @@
                     self.exception('localisation support error for language %s',
                                    language)
 
-    def appobjects_path(self):
-        """return a list of files or directories where the registry will look
-        for application objects
-        """
-        templpath = list(reversed(self.cubes_path()))
-        if self.apphome: # may be unset in tests
-            templpath.append(self.apphome)
-        return self.build_appobjects_path(templpath)
+    @staticmethod
+    def _sorted_appobjects(appobjects):
+        appobjects = sorted(appobjects)
+        try:
+            index = appobjects.index('entities')
+        except ValueError:
+            pass
+        else:
+            # put entities first
+            appobjects.insert(0, appobjects.pop(index))
+        return appobjects
+
+    def _appobjects_cube_modnames(self, cube):
+        modnames = []
+        cube_submodnames = self._sorted_appobjects(self.cube_appobject_path)
+        for name in cube_submodnames:
+            for modname, filepath in _expand_modname('.'.join(['cubes', cube, name])):
+                modnames.append(modname)
+        return modnames
+
+    def appobjects_modnames(self):
+        modnames = []
+        for name in self._sorted_appobjects(self.cubicweb_appobject_path):
+            for modname, filepath in _expand_modname('cubicweb.' + name):
+                modnames.append(modname)
+        for cube in reversed(self.cubes()):
+            modnames.extend(self._appobjects_cube_modnames(cube))
+        if self.apphome:
+            cube_submodnames = self._sorted_appobjects(self.cube_appobject_path)
+            apphome = realpath(self.apphome)
+            for name in cube_submodnames:
+                for modname, filepath in _expand_modname(name):
+                    # ensure file is in apphome
+                    if realpath(filepath).startswith(apphome):
+                        modnames.append(modname)
+        return modnames
 
     def set_sources_mode(self, sources):
         if not 'all' in sources:
diff -r bf6106b91633 -r 8496135b6dc1 cubicweb/cwvreg.py
--- a/cubicweb/cwvreg.py	Thu Jan 19 15:27:39 2017 +0100
+++ b/cubicweb/cwvreg.py	Thu Jan 19 14:53:05 2017 +0100
@@ -29,7 +29,7 @@
 
 from logilab.common.decorators import cached, clear_cache
 from logilab.common.deprecation import class_deprecated
-from logilab.common.modutils import cleanup_sys_modules
+from logilab.common.modutils import clean_sys_modules
 from logilab.common.registry import RegistryStore, Registry, ObjectNotFound, RegistryNotFound
 
 from rql import RQLHelper
@@ -417,7 +417,7 @@
         """set instance'schema and load application objects"""
         self._set_schema(schema)
         # now we can load application's web objects
-        self.reload(self.config.appobjects_path(), force_reload=False)
+        self.reload(self.config.appobjects_modnames(), force_reload=False)
         # map lowered entity type names to their actual name
         self.case_insensitive_etypes = {}
         for eschema in self.schema.entities():
@@ -426,13 +426,28 @@
             clear_cache(eschema, 'ordered_relations')
             clear_cache(eschema, 'meta_attributes')
 
+    def is_reload_needed(self, modnames):
+        """overriden to handle modules names instead of directories"""
+        lastmodifs = self._lastmodifs
+        for modname in modnames:
+            if modname not in sys.modules:
+                # new module to load
+                return True
+            filepath = sys.modules[modname].__file__
+            if filepath.endswith('.py'):
+                mdate = self._mdate(filepath)
+                if filepath not in lastmodifs or lastmodifs[filepath] < mdate:
+                    self.info('File %s changed since last visit', filepath)
+                    return True
+        return False
+
     def reload_if_needed(self):
-        path = self.config.appobjects_path()
-        if self.is_reload_needed(path):
-            self.reload(path)
+        modnames = self.config.appobjects_modnames()
+        if self.is_reload_needed(modnames):
+            self.reload(modnames)
 
-    def _cleanup_sys_modules(self, path):
-        """Remove submodules of `directories` from `sys.modules` and cleanup
+    def _cleanup_sys_modules(self, modnames):
+        """Remove modules and submodules of `modnames` from `sys.modules` and cleanup
         CW_EVENT_MANAGER accordingly.
 
         We take care to properly remove obsolete registry callbacks.
@@ -446,18 +461,18 @@
                 # for non-function callable, we do nothing interesting
                 module = getattr(func, '__module__', None)
                 caches[id(callback)] = module
-        deleted_modules = set(cleanup_sys_modules(path))
+        deleted_modules = set(clean_sys_modules(modnames))
         for callbacklist in callbackdata:
             for callback in callbacklist[:]:
                 module = caches[id(callback)]
                 if module and module in deleted_modules:
                     callbacklist.remove(callback)
 
-    def reload(self, path, force_reload=True):
+    def reload(self, modnames, force_reload=True):
         """modification detected, reset and reload the vreg"""
         CW_EVENT_MANAGER.emit('before-registry-reload')
         if force_reload:
-            self._cleanup_sys_modules(path)
+            self._cleanup_sys_modules(modnames)
             cubes = self.config.cubes()
             # if the fs code use some cubes not yet registered into the instance
             # we should cleanup sys.modules for those as well to avoid potential
@@ -465,9 +480,9 @@
             cfg = self.config
             for cube in cfg.expand_cubes(cubes, with_recommends=True):
                 if not cube in cubes:
-                    cpath = cfg.build_appobjects_cube_path([cfg.cube_dir(cube)])
-                    self._cleanup_sys_modules(cpath)
-        self.register_objects(path)
+                    cube_modnames = cfg.appobjects_cube_modnames(cube)
+                    self._cleanup_sys_modules(cube_modnames)
+        self.register_modnames(modnames)
         CW_EVENT_MANAGER.emit('after-registry-reload')
 
     def load_file(self, filepath, modname):
diff -r bf6106b91633 -r 8496135b6dc1 cubicweb/devtools/devctl.py
--- a/cubicweb/devtools/devctl.py	Thu Jan 19 15:27:39 2017 +0100
+++ b/cubicweb/devtools/devctl.py	Thu Jan 19 14:53:05 2017 +0100
@@ -37,6 +37,7 @@
 from six.moves import input
 
 from logilab.common import STD_BLACKLIST
+from logilab.common.modutils import clean_sys_modules
 from logilab.common.fileutils import ensure_fs_mode
 from logilab.common.shellutils import find
 
@@ -100,24 +101,6 @@
         return None
 
 
-def cleanup_sys_modules(config):
-    # cleanup sys.modules, required when we're updating multiple cubes
-    appobjects_path = config.appobjects_path()
-    for name, mod in list(sys.modules.items()):
-        if mod is None:
-            # duh ? logilab.common.os for instance
-            del sys.modules[name]
-            continue
-        if not hasattr(mod, '__file__'):
-            continue
-        if mod.__file__ is None:
-            # odd/rare but real
-            continue
-        for path in appobjects_path:
-            if mod.__file__.startswith(path):
-                del sys.modules[name]
-                break
-
 def generate_schema_pot(w, cubedir=None):
     """generate a pot file with schema specific i18n messages
 
@@ -136,7 +119,7 @@
     else:
         config = DevConfiguration()
         cube = libconfig = None
-    cleanup_sys_modules(config)
+    clean_sys_modules(config.appobjects_modnames())
     schema = config.load_schema(remove_unused_rtypes=False)
     vreg = CWRegistryStore(config)
     # set_schema triggers objects registrations
@@ -161,7 +144,7 @@
         # (cubicweb incl.)
         from cubicweb.cwvreg import CWRegistryStore
         libschema = libconfig.load_schema(remove_unused_rtypes=False)
-        cleanup_sys_modules(libconfig)
+        clean_sys_modules(libconfig.appobjects_modnames())
         libvreg = CWRegistryStore(libconfig)
         libvreg.set_schema(libschema) # trigger objects registration
         libafss = libvreg['uicfg']['autoform_section']
diff -r bf6106b91633 -r 8496135b6dc1 cubicweb/test/unittest_cwconfig.py
--- a/cubicweb/test/unittest_cwconfig.py	Thu Jan 19 15:27:39 2017 +0100
+++ b/cubicweb/test/unittest_cwconfig.py	Thu Jan 19 14:53:05 2017 +0100
@@ -178,17 +178,6 @@
         self.assertEqual(self.config.expand_cubes(('email', 'comment')),
                           ['email', 'comment', 'file'])
 
-    def test_appobjects_path(self):
-        path = [unabsolutize(p) for p in self.config.appobjects_path()]
-        self.assertEqual(path[0], 'entities')
-        self.assertCountEqual(path[1:4], ['web/views', 'sobjects', 'hooks'])
-        self.assertEqual(path[4], 'file/entities')
-        self.assertCountEqual(path[5:7],
-                              ['file/views.py', 'file/hooks'])
-        self.assertEqual(path[7], 'email/entities.py')
-        self.assertCountEqual(path[8:10],
-                              ['email/views', 'email/hooks.py'])
-        self.assertEqual(path[10:], ['test/data/entities.py', 'test/data/views.py'])
 
     def test_init_cubes_ignore_pyramid_cube(self):
         warning_msg = 'cubicweb-pyramid got integrated into CubicWeb'
@@ -463,7 +452,48 @@
                              join(libdir, 'schema.py'))
             self.assertEqual(config.schema_modnames(), expected)
 
-
+    @templibdir
+    def test_appobjects_modnames(self, libdir):
+        for filepath in (
+            join(libdir, 'entities.py'),
+            join(libdir, 'cubicweb_foo', '__init__.py'),
+            join(libdir, 'cubicweb_foo', 'entities', '__init__.py'),
+            join(libdir, 'cubicweb_foo', 'entities', 'a.py'),
+            join(libdir, 'cubicweb_foo', 'hooks.py'),
+            join(libdir, 'cubes', '__init__.py'),
+            join(libdir, 'cubes', 'bar', '__init__.py'),
+            join(libdir, 'cubes', 'bar', 'hooks.py'),
+            join(libdir, '_instance_dir', 'data1', 'entities.py'),
+            join(libdir, '_instance_dir', 'data2', 'hooks.py'),
+        ):
+            create_filepath(filepath)
+        instance_dir, cubes_dir = (
+            join(libdir, '_instance_dir'), join(libdir, 'cubes'))
+        expected = [
+            'cubicweb.entities',
+            'cubicweb.entities.adapters',
+            'cubicweb.entities.authobjs',
+            'cubicweb.entities.lib',
+            'cubicweb.entities.schemaobjs',
+            'cubicweb.entities.sources',
+            'cubicweb.entities.wfobjs',
+            'cubes.bar.hooks',
+            'cubes.foo.entities',
+            'cubes.foo.entities.a',
+            'cubes.foo.hooks',
+        ]
+        # data1 has entities
+        with temp_config('data1', instance_dir, cubes_dir,
+                         ('foo', 'bar')) as config:
+            config.cube_appobject_path = set(['entities', 'hooks'])
+            self.assertEqual(config.appobjects_modnames(),
+                             expected + ['entities'])
+        # data2 has hooks
+        with temp_config('data2', instance_dir, cubes_dir,
+                         ('foo', 'bar')) as config:
+            config.cube_appobject_path = set(['entities', 'hooks'])
+            self.assertEqual(config.appobjects_modnames(),
+                             expected + ['hooks'])
 
 
 if __name__ == '__main__':
diff -r bf6106b91633 -r 8496135b6dc1 cubicweb/web/webconfig.py
--- a/cubicweb/web/webconfig.py	Thu Jan 19 15:27:39 2017 +0100
+++ b/cubicweb/web/webconfig.py	Thu Jan 19 14:53:05 2017 +0100
@@ -82,7 +82,7 @@
     """the WebConfiguration is a singleton object handling instance's
     configuration and preferences
     """
-    cubicweb_appobject_path = CubicWebConfiguration.cubicweb_appobject_path | set([join('web', 'views')])
+    cubicweb_appobject_path = CubicWebConfiguration.cubicweb_appobject_path | set(['web.views'])
     cube_appobject_path = CubicWebConfiguration.cube_appobject_path | set(['views'])
 
     options = merge_options(CubicWebConfiguration.options + (