# HG changeset patch
# User Denis Laxalde <denis.laxalde@logilab.fr>
# Date 1473754560 -7200
# Node ID bc04039acd2eafa98d4ecc64d436130f78f75352
# Parent  013d16661c7f15e61d2fdb6c288dbd708eb581e6
[config] Make available_cubes aware of cubes installed as packages

For this add a "cubicweb.cubes" entry points group which will be scanned
through to find out installed cubes. Entries in this group are expected to
expose the module name of a cube (i.e. `cubicweb_foo`).

Note that CubicWebConfiguration's available_cubes method will return the
module name of cubes as packages (cubicweb_foo), so we had to add a special
"key" sorting function to keep cubes sorted as before, despite possible
different distribution schemes. This makes it possible to handle loading of
CTL plugins in an almost similar manner as before (just tweaking the package
name from cube name in load_cwctl_plugins method).

I had to tweak (again?) the test_cubes_path method in unittest_cwconfig.py but
did not find out why.

Apart from unforeseen bugs and pending documentation, this finishes the work
on porting cubes to standard Python packages. Closes #13001466.

diff -r 013d16661c7f -r bc04039acd2e cubicweb/cwconfig.py
--- a/cubicweb/cwconfig.py	Wed Sep 14 17:12:37 2016 +0200
+++ b/cubicweb/cwconfig.py	Tue Sep 13 10:16:00 2016 +0200
@@ -188,6 +188,8 @@
 from os.path import (exists, join, expanduser, abspath, normpath,
                      basename, isdir, dirname, splitext)
 import pkgutil
+import pkg_resources
+import re
 from smtplib import SMTP
 import stat
 import sys
@@ -265,6 +267,12 @@
     return prefix
 
 
+def _cube_pkgname(cube):
+    if not cube.startswith('cubicweb_'):
+        return 'cubicweb_' + cube
+    return cube
+
+
 # persistent options definition
 PERSISTENT_OPTIONS = (
     ('encoding',
@@ -447,9 +455,21 @@
 
     @classmethod
     def available_cubes(cls):
-        cls.warning('only listing "legacy" cubes found in cubes path')
-        import re
         cubes = set()
+        for entry_point in pkg_resources.iter_entry_points(
+                group='cubicweb.cubes', name=None):
+            try:
+                module = entry_point.load()
+            except ImportError:
+                continue
+            else:
+                modname = module.__name__
+                if not modname.startswith('cubicweb_'):
+                    cls.warning('entry point %s does not appear to be a cube',
+                                entry_point)
+                    continue
+                cubes.add(modname)
+        # Legacy cubes.
         for directory in cls.cubes_search_path():
             if not exists(directory):
                 cls.error('unexistant directory in cubes search path: %s'
@@ -463,7 +483,18 @@
                 cubedir = join(directory, cube)
                 if isdir(cubedir) and exists(join(cubedir, '__init__.py')):
                     cubes.add(cube)
-        return sorted(cubes)
+
+        def sortkey(cube):
+            """Preserve sorting with "cubicweb_" prefix."""
+            prefix = 'cubicweb_'
+            if cube.startswith(prefix):
+                # add a suffix to have a deterministic sorting between
+                # 'cubicweb_<cube>' and '<cube>' (useful in tests with "hash
+                # randomization" turned on).
+                return cube[len(prefix):] + '~'
+            return cube
+
+        return sorted(cubes, key=sortkey)
 
     @classmethod
     def cubes_search_path(cls):
@@ -487,7 +518,8 @@
         """return the cube directory for the given cube id, raise
         `ConfigurationError` if it doesn't exist
         """
-        loader = pkgutil.find_loader('cubicweb_%s' % cube)
+        pkgname = _cube_pkgname(cube)
+        loader = pkgutil.find_loader(pkgname)
         if loader:
             return dirname(loader.get_filename())
         # Legacy cubes.
@@ -495,9 +527,9 @@
             cubedir = join(directory, cube)
             if exists(cubedir):
                 return cubedir
-        msg = ('no module cubicweb_%(cube)s in search path '
-               'nor cube %(cube)r in %(path)s')
+        msg = 'no module %(pkg)s in search path nor cube %(cube)r in %(path)s'
         raise ConfigurationError(msg % {'cube': cube,
+                                        'pkg': _cube_pkgname(cube),
                                         'path': cls.cubes_search_path()})
 
     @classmethod
@@ -508,8 +540,9 @@
     @classmethod
     def cube_pkginfo(cls, cube):
         """return the information module for the given cube"""
+        pkgname = _cube_pkgname(cube)
         try:
-            return importlib.import_module('cubicweb_%s.__pkginfo__' % cube)
+            return importlib.import_module('%s.__pkginfo__' % pkgname)
         except ImportError:
             cube = CW_MIGRATION_MAP.get(cube, cube)
             try:
@@ -649,17 +682,22 @@
                 continue
             cls.info('loaded cubicweb-ctl plugin %s', ctlmod)
         for cube in cls.available_cubes():
-            pluginfile = join(cls.cube_dir(cube), 'ccplugin.py')
-            initfile = join(cls.cube_dir(cube), '__init__.py')
+            cubedir = cls.cube_dir(cube)
+            pluginfile = join(cubedir, 'ccplugin.py')
+            initfile = join(cubedir, '__init__.py')
+            if cube.startswith('cubicweb_'):
+                pkgname = cube
+            else:
+                pkgname = 'cubes.%s' % cube
             if exists(pluginfile):
                 try:
-                    __import__('cubes.%s.ccplugin' % cube)
+                    __import__(pkgname + '.ccplugin')
                     cls.info('loaded cubicweb-ctl plugin from %s', cube)
                 except Exception:
                     cls.exception('while loading plugin %s', pluginfile)
             elif exists(initfile):
                 try:
-                    __import__('cubes.%s' % cube)
+                    __import__(pkgname)
                 except Exception:
                     cls.exception('while loading cube %s', cube)
             else:
@@ -841,7 +879,7 @@
         # load cubes'__init__.py file first
         for cube in cubes:
             try:
-                importlib.import_module('cubicweb_%s' % cube)
+                importlib.import_module(_cube_pkgname(cube))
             except ImportError:
                 # Legacy cube.
                 __import__('cubes.%s' % cube)
diff -r 013d16661c7f -r bc04039acd2e cubicweb/skeleton/setup.py.tmpl
--- a/cubicweb/skeleton/setup.py.tmpl	Wed Sep 14 17:12:37 2016 +0200
+++ b/cubicweb/skeleton/setup.py.tmpl	Tue Sep 13 10:16:00 2016 +0200
@@ -72,5 +72,10 @@
     classifiers=classifiers,
     packages=find_packages(exclude=['test']),
     install_requires=install_requires,
+    entry_points={
+        'cubicweb.cubes': [
+            '%(cubename)s=cubicweb_%(cubename)s',
+        ],
+    },
     zip_safe=False,
 )
diff -r 013d16661c7f -r bc04039acd2e cubicweb/test/requirements.txt
--- a/cubicweb/test/requirements.txt	Wed Sep 14 17:12:37 2016 +0200
+++ b/cubicweb/test/requirements.txt	Tue Sep 13 10:16:00 2016 +0200
@@ -1,4 +1,5 @@
 Pygments
+mock
 #fyzz XXX pip install fails
 cubicweb-card
 cubicweb-file
diff -r 013d16661c7f -r bc04039acd2e cubicweb/test/unittest_cwconfig.py
--- a/cubicweb/test/unittest_cwconfig.py	Wed Sep 14 17:12:37 2016 +0200
+++ b/cubicweb/test/unittest_cwconfig.py	Tue Sep 13 10:16:00 2016 +0200
@@ -21,8 +21,10 @@
 import os
 import tempfile
 from os.path import dirname, join, abspath
+from pkg_resources import EntryPoint, Distribution
 import unittest
 
+from mock import patch
 from six import PY3
 
 from logilab.common.modutils import cleanup_sys_modules
@@ -60,6 +62,34 @@
     def tearDown(self):
         ApptestConfiguration.CUBES_PATH = []
 
+    def iter_entry_points(group, name):
+        """Mock pkg_resources.iter_entry_points to yield EntryPoint from
+        packages found in test/data/libpython even though these are not
+        installed.
+        """
+        libpython = CubicWebConfigurationTC.datapath('libpython')
+        prefix = 'cubicweb_'
+        for pkgname in os.listdir(libpython):
+            if not pkgname.startswith(prefix):
+                continue
+            location = join(libpython, pkgname)
+            yield EntryPoint(pkgname[len(prefix):], pkgname,
+                             dist=Distribution(location))
+
+    @patch('pkg_resources.iter_entry_points', side_effect=iter_entry_points)
+    def test_available_cubes(self, mock_iter_entry_points):
+        expected_cubes = [
+            'card', 'cubicweb_comment', 'cubicweb_email', 'file',
+            'cubicweb_file', 'cubicweb_forge', 'localperms',
+            'cubicweb_mycube', 'tag',
+        ]
+        self._test_available_cubes(expected_cubes)
+        mock_iter_entry_points.assert_called_once_with(
+            group='cubicweb.cubes', name=None)
+
+    def _test_available_cubes(self, expected_cubes):
+        self.assertEqual(self.config.available_cubes(), expected_cubes)
+
     def test_reorder_cubes(self):
         # forge depends on email and file and comment
         # email depends on file
@@ -133,6 +163,15 @@
     def tearDown(self):
         ApptestConfiguration.CUBES_PATH = []
 
+    def test_available_cubes(self):
+        expected_cubes = sorted(set([
+            # local cubes
+            'comment', 'email', 'file', 'forge', 'mycube',
+            # test dependencies
+            'card', 'file', 'localperms', 'tag',
+        ]))
+        self._test_available_cubes(expected_cubes)
+
     def test_reorder_cubes_recommends(self):
         from cubes.comment import __pkginfo__ as comment_pkginfo
         self._test_reorder_cubes_recommends(comment_pkginfo)
@@ -159,8 +198,7 @@
         from cubes import mycube
         self.assertEqual(mycube.__path__, [join(self.custom_cubes_dir, 'mycube')])
         # file cube should be overriden by the one found in data/cubes
-        sys.modules.pop('cubes.file', None)
-        if PY3:
+        if sys.modules.pop('cubes.file', None) and PY3:
             del cubes.file
         from cubes import file
         self.assertEqual(file.__path__, [join(self.custom_cubes_dir, 'file')])