# HG changeset patch
# User Philippe Pepiot <philippe.pepiot@logilab.fr>
# Date 1492183032 -7200
# Node ID 127f7da447a4ced3a61c912d597aa8b97fcb5156
# Parent  c1d7c0f662108a1e8f36f0fdb6d871908cc6d712
[cwconfig] load appobjects subpackages recursively

This is a regression appeared in 3.25.0. Appobjects (entities, views etc)
subpackages should be loaded recursively, this is the historical behavior,
relevant code is in logilab.common.registry._toload_info().
To keep historical behavior schema subpackages should not be loaded, relevant
code is in yams.reader.get_schema_files().
We may want to have a consistent behavior later on...

Closes #17073706.

diff -r c1d7c0f66210 -r 127f7da447a4 cubicweb/cwconfig.py
--- a/cubicweb/cwconfig.py	Fri Apr 14 15:32:17 2017 +0200
+++ b/cubicweb/cwconfig.py	Fri Apr 14 17:17:12 2017 +0200
@@ -285,12 +285,15 @@
         return 'cubes.' + cube
 
 
-def _expand_modname(modname):
-    """expand modules names `modname` if exists by walking non package submodules
-    and yield (submodname, filepath) including `modname` itself
+def _expand_modname(modname, recursive=True):
+    """expand modules names `modname` if exists by recursively walking
+    submodules and subpackages and yield (submodname, filepath) including
+    `modname` itself
 
     If the file ends with .pyc or .pyo (python bytecode) also check that the
     corresponding source .py file exists before yielding.
+
+    If `recursive` is False skip subpackages.
     """
     try:
         loader = pkgutil.find_loader(modname)
@@ -312,11 +315,14 @@
     if loader.is_package(modname):
         path = dirname(filepath)
         for subloader, subname, ispkg in pkgutil.walk_packages([path]):
-            # ignore subpackages (historical behavior)
+            submodname = '.'.join([modname, subname])
             if not ispkg:
                 filepath = subloader.find_module(subname).get_filename()
                 if check_source_file(filepath):
-                    yield modname + '.' + subname, filepath
+                    yield submodname, filepath
+            elif recursive:
+                for x in _expand_modname(submodname, recursive=True):
+                    yield x
 
 
 # persistent options definition
@@ -819,11 +825,13 @@
             modnames.append(('cubicweb', 'cubicweb.schemas.' + name))
         for cube in reversed(self.cubes()):
             for modname, filepath in _expand_modname(
-                    '{0}.schema'.format(_cube_modname(cube))):
+                    '{0}.schema'.format(_cube_modname(cube)),
+                    recursive=False):
                 modnames.append((cube, modname))
         if self.apphome:
             apphome = realpath(self.apphome)
-            for modname, filepath in _expand_modname('schema'):
+            for modname, filepath in _expand_modname(
+                    'schema', recursive=False):
                 if realpath(filepath).startswith(apphome):
                     modnames.append(('data', modname))
         return modnames
diff -r c1d7c0f66210 -r 127f7da447a4 cubicweb/test/unittest_cwconfig.py
--- a/cubicweb/test/unittest_cwconfig.py	Fri Apr 14 15:32:17 2017 +0200
+++ b/cubicweb/test/unittest_cwconfig.py	Fri Apr 14 17:17:12 2017 +0200
@@ -409,20 +409,32 @@
         self.assertEqual(list(_expand_modname('lib.a')), [
             ('lib.a', join(tempdir, 'a.py')),
         ])
-        # lib.b.d (subpackage) not to be imported
+        # lib.b.d should be imported
         self.assertEqual(list(_expand_modname('lib.b')), [
             ('lib.b', join(tempdir, 'b', '__init__.py')),
             ('lib.b.a', join(tempdir, 'b', 'a.py')),
             ('lib.b.c', join(tempdir, 'b', 'c.py')),
+            ('lib.b.d', join(tempdir, 'b', 'd', '__init__.py')),
+        ])
+        # lib.b.d should not be imported without recursive mode
+        self.assertEqual(list(_expand_modname('lib.b', recursive=False)), [
+            ('lib.b', join(tempdir, 'b', '__init__.py')),
+            ('lib.b.a', join(tempdir, 'b', 'a.py')),
+            ('lib.b.c', join(tempdir, 'b', 'c.py')),
         ])
         self.assertEqual(list(_expand_modname('lib')), [
             ('lib', join(tempdir, '__init__.py')),
             ('lib.a', join(tempdir, 'a.py')),
+            ('lib.b', join(tempdir, 'b', '__init__.py')),
+            ('lib.b.a', join(tempdir, 'b', 'a.py')),
+            ('lib.b.c', join(tempdir, 'b', 'c.py')),
+            ('lib.b.d', join(tempdir, 'b', 'd', '__init__.py')),
             ('lib.c', join(tempdir, 'c.py')),
         ])
         for source in (
             join(tempdir, 'c.py'),
             join(tempdir, 'b', 'c.py'),
+            join(tempdir, 'b', 'd', '__init__.py'),
         ):
             if not PY3:
                 # ensure pyc file exists.
@@ -441,6 +453,8 @@
         self.assertEqual(list(_expand_modname('lib')), [
             ('lib', join(tempdir, '__init__.py')),
             ('lib.a', join(tempdir, 'a.py')),
+            ('lib.b', join(tempdir, 'b', '__init__.py')),
+            ('lib.b.a', join(tempdir, 'b', 'a.py')),
         ])
 
     @templibdir
@@ -451,6 +465,8 @@
             join(libdir, 'cubicweb_foo', 'schema', '__init__.py'),
             join(libdir, 'cubicweb_foo', 'schema', 'a.py'),
             join(libdir, 'cubicweb_foo', 'schema', 'b.py'),
+            # subpackages should not be loaded
+            join(libdir, 'cubicweb_foo', 'schema', 'c', '__init__.py'),
             join(libdir, 'cubes', '__init__.py'),
             join(libdir, 'cubes', 'bar', '__init__.py'),
             join(libdir, 'cubes', 'bar', 'schema.py'),
@@ -492,6 +508,10 @@
             join(libdir, 'cubicweb_foo', '__init__.py'),
             join(libdir, 'cubicweb_foo', 'entities', '__init__.py'),
             join(libdir, 'cubicweb_foo', 'entities', 'a.py'),
+            # subpackages should be loaded recursively
+            join(libdir, 'cubicweb_foo', 'entities', 'b', '__init__.py'),
+            join(libdir, 'cubicweb_foo', 'entities', 'b', 'a.py'),
+            join(libdir, 'cubicweb_foo', 'entities', 'b', 'c', '__init__.py'),
             join(libdir, 'cubicweb_foo', 'hooks.py'),
             join(libdir, 'cubes', '__init__.py'),
             join(libdir, 'cubes', 'bar', '__init__.py'),
@@ -513,6 +533,9 @@
             'cubes.bar.hooks',
             'cubicweb_foo.entities',
             'cubicweb_foo.entities.a',
+            'cubicweb_foo.entities.b',
+            'cubicweb_foo.entities.b.a',
+            'cubicweb_foo.entities.b.c',
             'cubicweb_foo.hooks',
         ]
         # data1 has entities