[config] Make available_cubes aware of cubes installed as packages
For this add a "cubicweb.cubes" entry points group which will be scanned
through to find out installed cubes. Entries in this group are expected to
expose the module name of a cube (i.e. `cubicweb_foo`).
Note that CubicWebConfiguration's available_cubes method will return the
module name of cubes as packages (cubicweb_foo), so we had to add a special
"key" sorting function to keep cubes sorted as before, despite possible
different distribution schemes. This makes it possible to handle loading of
CTL plugins in an almost similar manner as before (just tweaking the package
name from cube name in load_cwctl_plugins method).
I had to tweak (again?) the test_cubes_path method in unittest_cwconfig.py but
did not find out why.
Apart from unforeseen bugs and pending documentation, this finishes the work
on porting cubes to standard Python packages. Closes #13001466.
--- a/cubicweb/cwconfig.py Wed Sep 14 17:12:37 2016 +0200
+++ b/cubicweb/cwconfig.py Tue Sep 13 10:16:00 2016 +0200
@@ -188,6 +188,8 @@
from os.path import (exists, join, expanduser, abspath, normpath,
basename, isdir, dirname, splitext)
import pkgutil
+import pkg_resources
+import re
from smtplib import SMTP
import stat
import sys
@@ -265,6 +267,12 @@
return prefix
+def _cube_pkgname(cube):
+ if not cube.startswith('cubicweb_'):
+ return 'cubicweb_' + cube
+ return cube
+
+
# persistent options definition
PERSISTENT_OPTIONS = (
('encoding',
@@ -447,9 +455,21 @@
@classmethod
def available_cubes(cls):
- cls.warning('only listing "legacy" cubes found in cubes path')
- import re
cubes = set()
+ for entry_point in pkg_resources.iter_entry_points(
+ group='cubicweb.cubes', name=None):
+ try:
+ module = entry_point.load()
+ except ImportError:
+ continue
+ else:
+ modname = module.__name__
+ if not modname.startswith('cubicweb_'):
+ cls.warning('entry point %s does not appear to be a cube',
+ entry_point)
+ continue
+ cubes.add(modname)
+ # Legacy cubes.
for directory in cls.cubes_search_path():
if not exists(directory):
cls.error('unexistant directory in cubes search path: %s'
@@ -463,7 +483,18 @@
cubedir = join(directory, cube)
if isdir(cubedir) and exists(join(cubedir, '__init__.py')):
cubes.add(cube)
- return sorted(cubes)
+
+ def sortkey(cube):
+ """Preserve sorting with "cubicweb_" prefix."""
+ prefix = 'cubicweb_'
+ if cube.startswith(prefix):
+ # add a suffix to have a deterministic sorting between
+ # 'cubicweb_<cube>' and '<cube>' (useful in tests with "hash
+ # randomization" turned on).
+ return cube[len(prefix):] + '~'
+ return cube
+
+ return sorted(cubes, key=sortkey)
@classmethod
def cubes_search_path(cls):
@@ -487,7 +518,8 @@
"""return the cube directory for the given cube id, raise
`ConfigurationError` if it doesn't exist
"""
- loader = pkgutil.find_loader('cubicweb_%s' % cube)
+ pkgname = _cube_pkgname(cube)
+ loader = pkgutil.find_loader(pkgname)
if loader:
return dirname(loader.get_filename())
# Legacy cubes.
@@ -495,9 +527,9 @@
cubedir = join(directory, cube)
if exists(cubedir):
return cubedir
- msg = ('no module cubicweb_%(cube)s in search path '
- 'nor cube %(cube)r in %(path)s')
+ msg = 'no module %(pkg)s in search path nor cube %(cube)r in %(path)s'
raise ConfigurationError(msg % {'cube': cube,
+ 'pkg': _cube_pkgname(cube),
'path': cls.cubes_search_path()})
@classmethod
@@ -508,8 +540,9 @@
@classmethod
def cube_pkginfo(cls, cube):
"""return the information module for the given cube"""
+ pkgname = _cube_pkgname(cube)
try:
- return importlib.import_module('cubicweb_%s.__pkginfo__' % cube)
+ return importlib.import_module('%s.__pkginfo__' % pkgname)
except ImportError:
cube = CW_MIGRATION_MAP.get(cube, cube)
try:
@@ -649,17 +682,22 @@
continue
cls.info('loaded cubicweb-ctl plugin %s', ctlmod)
for cube in cls.available_cubes():
- pluginfile = join(cls.cube_dir(cube), 'ccplugin.py')
- initfile = join(cls.cube_dir(cube), '__init__.py')
+ cubedir = cls.cube_dir(cube)
+ pluginfile = join(cubedir, 'ccplugin.py')
+ initfile = join(cubedir, '__init__.py')
+ if cube.startswith('cubicweb_'):
+ pkgname = cube
+ else:
+ pkgname = 'cubes.%s' % cube
if exists(pluginfile):
try:
- __import__('cubes.%s.ccplugin' % cube)
+ __import__(pkgname + '.ccplugin')
cls.info('loaded cubicweb-ctl plugin from %s', cube)
except Exception:
cls.exception('while loading plugin %s', pluginfile)
elif exists(initfile):
try:
- __import__('cubes.%s' % cube)
+ __import__(pkgname)
except Exception:
cls.exception('while loading cube %s', cube)
else:
@@ -841,7 +879,7 @@
# load cubes'__init__.py file first
for cube in cubes:
try:
- importlib.import_module('cubicweb_%s' % cube)
+ importlib.import_module(_cube_pkgname(cube))
except ImportError:
# Legacy cube.
__import__('cubes.%s' % cube)
--- a/cubicweb/skeleton/setup.py.tmpl Wed Sep 14 17:12:37 2016 +0200
+++ b/cubicweb/skeleton/setup.py.tmpl Tue Sep 13 10:16:00 2016 +0200
@@ -72,5 +72,10 @@
classifiers=classifiers,
packages=find_packages(exclude=['test']),
install_requires=install_requires,
+ entry_points={
+ 'cubicweb.cubes': [
+ '%(cubename)s=cubicweb_%(cubename)s',
+ ],
+ },
zip_safe=False,
)
--- a/cubicweb/test/requirements.txt Wed Sep 14 17:12:37 2016 +0200
+++ b/cubicweb/test/requirements.txt Tue Sep 13 10:16:00 2016 +0200
@@ -1,4 +1,5 @@
Pygments
+mock
#fyzz XXX pip install fails
cubicweb-card
cubicweb-file
--- a/cubicweb/test/unittest_cwconfig.py Wed Sep 14 17:12:37 2016 +0200
+++ b/cubicweb/test/unittest_cwconfig.py Tue Sep 13 10:16:00 2016 +0200
@@ -21,8 +21,10 @@
import os
import tempfile
from os.path import dirname, join, abspath
+from pkg_resources import EntryPoint, Distribution
import unittest
+from mock import patch
from six import PY3
from logilab.common.modutils import cleanup_sys_modules
@@ -60,6 +62,34 @@
def tearDown(self):
ApptestConfiguration.CUBES_PATH = []
+ def iter_entry_points(group, name):
+ """Mock pkg_resources.iter_entry_points to yield EntryPoint from
+ packages found in test/data/libpython even though these are not
+ installed.
+ """
+ libpython = CubicWebConfigurationTC.datapath('libpython')
+ prefix = 'cubicweb_'
+ for pkgname in os.listdir(libpython):
+ if not pkgname.startswith(prefix):
+ continue
+ location = join(libpython, pkgname)
+ yield EntryPoint(pkgname[len(prefix):], pkgname,
+ dist=Distribution(location))
+
+ @patch('pkg_resources.iter_entry_points', side_effect=iter_entry_points)
+ def test_available_cubes(self, mock_iter_entry_points):
+ expected_cubes = [
+ 'card', 'cubicweb_comment', 'cubicweb_email', 'file',
+ 'cubicweb_file', 'cubicweb_forge', 'localperms',
+ 'cubicweb_mycube', 'tag',
+ ]
+ self._test_available_cubes(expected_cubes)
+ mock_iter_entry_points.assert_called_once_with(
+ group='cubicweb.cubes', name=None)
+
+ def _test_available_cubes(self, expected_cubes):
+ self.assertEqual(self.config.available_cubes(), expected_cubes)
+
def test_reorder_cubes(self):
# forge depends on email and file and comment
# email depends on file
@@ -133,6 +163,15 @@
def tearDown(self):
ApptestConfiguration.CUBES_PATH = []
+ def test_available_cubes(self):
+ expected_cubes = sorted(set([
+ # local cubes
+ 'comment', 'email', 'file', 'forge', 'mycube',
+ # test dependencies
+ 'card', 'file', 'localperms', 'tag',
+ ]))
+ self._test_available_cubes(expected_cubes)
+
def test_reorder_cubes_recommends(self):
from cubes.comment import __pkginfo__ as comment_pkginfo
self._test_reorder_cubes_recommends(comment_pkginfo)
@@ -159,8 +198,7 @@
from cubes import mycube
self.assertEqual(mycube.__path__, [join(self.custom_cubes_dir, 'mycube')])
# file cube should be overriden by the one found in data/cubes
- sys.modules.pop('cubes.file', None)
- if PY3:
+ if sys.modules.pop('cubes.file', None) and PY3:
del cubes.file
from cubes import file
self.assertEqual(file.__path__, [join(self.custom_cubes_dir, 'file')])