add egg support with dependencies auto-installation
authorAlain Leufroy <alain.leufroy@logilab.fr>
Thu, 25 Mar 2010 20:33:03 +0100
changeset 5024 9e718abe3fde
parent 5023 70d5d54e8b81
child 5025 2826f5406201
add egg support with dependencies auto-installation
README
__pkginfo__.py
setup.py
skeleton/__pkginfo__.py.tmpl
skeleton/setup.py
test/data/cubes/file/__pkginfo__.py
--- a/README	Thu Mar 25 20:32:52 2010 +0100
+++ b/README	Thu Mar 25 20:33:03 2010 +0100
@@ -1,6 +1,15 @@
 CubicWeb semantic web framework
 ===============================
 
+CubicWeb is a entities / relations based knowledge management system
+developped at Logilab.
+
+This package contains:
+* a repository server
+* a RQL command line client to the repository
+* an adaptative modpython interface to the server
+* a bunch of other management tools
+
 Install
 -------
 
--- a/__pkginfo__.py	Thu Mar 25 20:32:52 2010 +0100
+++ b/__pkginfo__.py	Thu Mar 25 20:33:03 2010 +0100
@@ -1,36 +1,21 @@
 # pylint: disable-msg=W0622,C0103
 """cubicweb global packaging information for the cubicweb knowledge management
 software
+
 :license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses
 """
 
-distname = "cubicweb"
-modname = "cubicweb"
+modname = distname = "cubicweb"
 
 numversion = (3, 7, 1)
 version = '.'.join(str(num) for num in numversion)
 
-license = 'LGPL'
-copyright = '''Copyright (c) 2003-2010 LOGILAB S.A. (Paris, FRANCE).
-http://www.logilab.fr/ -- mailto:contact@logilab.fr'''
-
+description = "a repository of entities / relations for knowledge management"
 author = "Logilab"
 author_email = "contact@logilab.fr"
-
-short_desc = "a repository of entities / relations for knowledge management"
-long_desc = """CubicWeb is a entities / relations based knowledge management system
-developped at Logilab.
-
-This package contains:
-* a repository server
-* a RQL command line client to the repository
-* an adaptative modpython interface to the server
-* a bunch of other management tools
-"""
-
 web = 'http://www.cubicweb.org'
 ftp = 'ftp://ftp.logilab.org/pub/cubicweb'
-pyversions = ['2.5', '2.6']
+license = 'LGPL'
 
 classifiers = [
            'Environment :: Web Environment',
@@ -39,6 +24,34 @@
            'Programming Language :: JavaScript',
 ]
 
+__depends__ = {
+    'logilab-common': '>= 0.49.0',
+    'logilab-mtconverter': '>= 0.6.0',
+    'rql': '>= 0.26.0',
+    'yams': '>= 0.28.1',
+    'docutils': '>= 0.6',
+    #gettext                    # for xgettext, msgcat, etc...
+    # web dependancies
+    'simplejson': '>= 2.0.9',
+    'lxml': '',
+    'Twisted': '',
+    # XXX graphviz
+    # server dependencies
+    'logilab-database': '',
+    'pysqlite': '>= 2.5.5', # XXX install pysqlite2
+    }
+
+__recommends__ = {
+    'Pyro': '>= 3.9.1',
+    'PIL': '',                  # for captcha
+    'pycrypto': '',             # for crypto extensions
+    'fyzz': '>= 0.1.0',         # for sparql
+    'pysixt': '>= 0.1.0',       # XXX for pdf export
+    'python-gettext': '>= 1.0', # XXX for pdf export
+    'vobject': '>= 0.6.0',      # for ical view
+    #'Products.FCKeditor':'',
+    #'SimpleTAL':'>= 4.1.6',
+    }
 
 import sys
 from os import listdir, environ
@@ -49,55 +62,52 @@
            if not s.endswith('.bat')]
 include_dirs = [join('test', 'data'),
                 join('server', 'test', 'data'),
+                join('hooks', 'test', 'data'),
                 join('web', 'test', 'data'),
                 join('devtools', 'test', 'data'),
                 'skeleton']
 
 
-entities_dir = 'entities'
-schema_dir = 'schemas'
-sobjects_dir = 'sobjects'
-server_migration_dir = join('misc', 'migration')
-data_dir = join('web', 'data')
-wdoc_dir = join('web', 'wdoc')
-wdocimages_dir = join(wdoc_dir, 'images')
-views_dir = join('web', 'views')
-i18n_dir = 'i18n'
+_server_migration_dir = join('misc', 'migration')
+_data_dir = join('web', 'data')
+_wdoc_dir = join('web', 'wdoc')
+_wdocimages_dir = join(_wdoc_dir, 'images')
+_views_dir = join('web', 'views')
+_i18n_dir = 'i18n'
 
-if environ.get('APYCOT_ROOT'):
+_pyversion = '.'.join(str(num) for num in sys.version_info[0:2])
+if '--home' in sys.argv:
     # --home install
-    pydir = 'python'
+    pydir = 'python' + _pyversion
 else:
-    python_version = '.'.join(str(num) for num in sys.version_info[0:2])
-    pydir = join('python' + python_version, 'site-packages')
+    pydir = join('python' + _pyversion, 'site-packages')
 
 try:
     data_files = [
-        # common data
-        #[join('share', 'cubicweb', 'entities'),
-        # [join(entities_dir, filename) for filename in listdir(entities_dir)]],
         # server data
         [join('share', 'cubicweb', 'schemas'),
-         [join(schema_dir, filename) for filename in listdir(schema_dir)]],
-        #[join('share', 'cubicweb', 'sobjects'),
-        # [join(sobjects_dir, filename) for filename in listdir(sobjects_dir)]],
+         [join('schemas', filename) for filename in listdir('schemas')]],
         [join('share', 'cubicweb', 'migration'),
-         [join(server_migration_dir, filename)
-          for filename in listdir(server_migration_dir)]],
+         [join(_server_migration_dir, filename)
+          for filename in listdir(_server_migration_dir)]],
         # web data
         [join('share', 'cubicweb', 'cubes', 'shared', 'data'),
-         [join(data_dir, fname) for fname in listdir(data_dir) if not isdir(join(data_dir, fname))]],
+         [join(_data_dir, fname) for fname in listdir(_data_dir)
+          if not isdir(join(_data_dir, fname))]],
         [join('share', 'cubicweb', 'cubes', 'shared', 'data', 'timeline'),
-         [join(data_dir, 'timeline', fname) for fname in listdir(join(data_dir, 'timeline'))]],
+         [join(_data_dir, 'timeline', fname)
+          for fname in listdir(join(_data_dir, 'timeline'))]],
         [join('share', 'cubicweb', 'cubes', 'shared', 'wdoc'),
-         [join(wdoc_dir, fname) for fname in listdir(wdoc_dir) if not isdir(join(wdoc_dir, fname))]],
+         [join(_wdoc_dir, fname) for fname in listdir(_wdoc_dir)
+          if not isdir(join(_wdoc_dir, fname))]],
         [join('share', 'cubicweb', 'cubes', 'shared', 'wdoc', 'images'),
-         [join(wdocimages_dir, fname) for fname in listdir(wdocimages_dir)]],
-        # XXX: .pt install should be handled properly in a near future version
+         [join(_wdocimages_dir, fname) for fname in listdir(_wdocimages_dir)]],
+        [join('share', 'cubicweb', 'cubes', 'shared', 'i18n'),
+         [join(_i18n_dir, fname) for fname in listdir(_i18n_dir)]],
+        # XXX: drop .pt files
         [join('lib', pydir, 'cubicweb', 'web', 'views'),
-         [join(views_dir, fname) for fname in listdir(views_dir) if fname.endswith('.pt')]],
-        [join('share', 'cubicweb', 'cubes', 'shared', 'i18n'),
-         [join(i18n_dir, fname) for fname in listdir(i18n_dir)]],
+         [join(_views_dir, fname) for fname in listdir(_views_dir)
+          if fname.endswith('.pt')]],
         # skeleton
         ]
 except OSError:
--- a/setup.py	Thu Mar 25 20:32:52 2010 +0100
+++ b/setup.py	Thu Mar 25 20:33:03 2010 +0100
@@ -24,38 +24,44 @@
 import os
 import sys
 import shutil
-from distutils.core import setup
-from distutils.command import install_lib
 from os.path import isdir, exists, join, walk
 
+try:
+   if os.environ.get('NO_SETUPTOOLS'):
+      raise ImportError() # do as there is no setuptools
+   from setuptools import setup
+   from setuptools.command import install_lib
+   USE_SETUPTOOLS = True
+except ImportError:
+   from distutils.core import setup
+   from distutils.command import install_lib
+   USE_SETUPTOOLS = False
+
 # import required features
-from __pkginfo__ import modname, version, license, short_desc, long_desc, \
-     web, author, author_email
+from __pkginfo__ import modname, version, license, description, web, \
+     author, author_email
+
+if exists('README'):
+   long_description = file('README').read()
+
 # import optional features
-try:
-    from __pkginfo__ import distname
-except ImportError:
-    distname = modname
-try:
-    from __pkginfo__ import scripts
-except ImportError:
-    scripts = []
-try:
-    from __pkginfo__ import data_files
-except ImportError:
-    data_files = None
-try:
-    from __pkginfo__ import subpackage_of
-except ImportError:
-    subpackage_of = None
-try:
-    from __pkginfo__ import include_dirs
-except ImportError:
-    include_dirs = []
-try:
-    from __pkginfo__ import ext_modules
-except ImportError:
-    ext_modules = None
+import __pkginfo__
+if USE_SETUPTOOLS:
+   requires = {}
+   for entry in ("__depends__", "__recommends__"):
+      requires.update(getattr(__pkginfo__, entry, {}))
+   install_requires = [("%s %s" % (d, v and v or "")).strip()
+                       for d, v in requires.iteritems()]
+else:
+   install_requires = []
+
+distname = getattr(__pkginfo__, 'distname', modname)
+scripts = getattr(__pkginfo__, 'scripts', ())
+include_dirs = getattr(__pkginfo__, 'include_dirs', ())
+data_files = getattr(__pkginfo__, 'data_files', None)
+subpackage_of = getattr(__pkginfo__, 'subpackage_of', None)
+ext_modules = getattr(__pkginfo__, 'ext_modules', None)
+
 
 BASE_BLACKLIST = ('CVS', 'debian', 'dist', 'build', '__buildlog')
 IGNORED_EXTENSIONS = ('.pyc', '.pyo', '.elc')
@@ -92,7 +98,8 @@
 
 def export(from_dir, to_dir,
            blacklist=BASE_BLACKLIST,
-           ignore_ext=IGNORED_EXTENSIONS):
+           ignore_ext=IGNORED_EXTENSIONS,
+           verbose=True):
     """make a mirror of from_dir in to_dir, omitting directories and files
     listed in the black list
     """
@@ -111,7 +118,8 @@
                 continue
             src = '%s/%s' % (directory, filename)
             dest = to_dir + src[len(from_dir):]
-            print >> sys.stderr, src, '->', dest
+            if verbose:
+               print >> sys.stderr, src, '->', dest
             if os.path.isdir(src):
                 if not exists(dest):
                     os.mkdir(dest)
@@ -154,29 +162,31 @@
                 base = modname
             for directory in include_dirs:
                 dest = join(self.install_dir, base, directory)
-                export(directory, dest)
+                export(directory, dest, verbose=False)
 
 def install(**kwargs):
     """setup entry point"""
+    if not USE_SETUPTOOLS and '--install-layout=deb' in sys.argv and \
+           sys.versioninfo < (2, 5, 4):
+       sys.argv.remove('--install-layout=deb')
+       print "W: remove '--install-layout=deb' option"
     if subpackage_of:
         package = subpackage_of + '.' + modname
         kwargs['package_dir'] = {package : '.'}
         packages = [package] + get_packages(os.getcwd(), package)
+        if USE_SETUPTOOLS:
+            kwargs['namespace_packages'] = [subpackage_of]
     else:
         kwargs['package_dir'] = {modname : '.'}
         packages = [modname] + get_packages(os.getcwd(), modname)
     kwargs['packages'] = packages
-    return setup(name = distname,
-                 version = version,
-                 license =license,
-                 description = short_desc,
-                 long_description = long_desc,
-                 author = author,
-                 author_email = author_email,
-                 url = web,
-                 scripts = ensure_scripts(scripts),
-                 data_files=data_files,
+    return setup(name=distname, version=version, license=license, url=web,
+                 description=description, long_description=long_description,
+                 author=author, author_email=author_email,
+                 scripts=ensure_scripts(scripts), data_files=data_files,
                  ext_modules=ext_modules,
+                 install_requires=install_requires,
+                 #dependency_links=["http://alain:alain@intranet.logilab.fr/~alain/"],
                  cmdclass={'install_lib': MyInstallLib},
                  **kwargs
                  )
--- a/skeleton/__pkginfo__.py.tmpl	Thu Mar 25 20:32:52 2010 +0100
+++ b/skeleton/__pkginfo__.py.tmpl	Thu Mar 25 20:33:03 2010 +0100
@@ -8,14 +8,11 @@
 version = '.'.join(str(num) for num in numversion)
 
 license = 'LCL'
-copyright = '''Copyright (c) %(year)s %(author)s.
-%(author-web-site)s -- mailto:%(author-email)s'''
 
 author = '%(author)s'
 author_email = '%(author-email)s'
 
-short_desc = '%(shortdesc)s'
-long_desc = '''%(longdesc)s'''
+description = '%(shortdesc)s'
 
 web = 'http://www.cubicweb.org/project/%%s' %% distname
 
@@ -43,8 +40,8 @@
 # Note: here, you'll need to add subdirectories if you want
 # them to be included in the debian package
 
+__depends__ = {'cubicweb': '>= 3.6.0'}
 __depends_cubes__ = {}
-__depends__ = {'cubicweb': '>= 3.6.0'}
-__use__ = (%(dependancies)s)
-__recommend__ = ()
+__recommends__ = {}
+__recommends_cubes__ = {}
 
--- a/skeleton/setup.py	Thu Mar 25 20:32:52 2010 +0100
+++ b/skeleton/setup.py	Thu Mar 25 20:33:03 2010 +0100
@@ -1,14 +1,12 @@
 #!/usr/bin/env python
-"""
+"""Generic Setup script, takes package info from __pkginfo__.py file
 
 :organization: Logilab
 :copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2.
 :contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr
 :license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses
 """
-# pylint: disable-msg=W0404,W0622,W0704,W0613,W0152
-# Copyright (c) 2003-2010 LOGILAB S.A. (Paris, FRANCE).
-# http://www.logilab.fr/ -- mailto:contact@logilab.fr
+# pylint: disable-msg=W0142,W0403,W0404,W0613,W0622,W0622,W0704,R0904,C0103,E0611
 #
 # This program is free software; you can redistribute it and/or modify it under
 # the terms of the GNU General Public License as published by the Free Software
@@ -22,36 +20,176 @@
 # You should have received a copy of the GNU General Public License along with
 # this program; if not, write to the Free Software Foundation, Inc.,
 # 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-""" Generic Setup script, takes package info from __pkginfo__.py file """
+
+import os
+import sys
+import shutil
+from os.path import isdir, exists, join, walk
 
-from distutils.core import setup
+try:
+   if os.environ.get('NO_SETUPTOOLS'):
+      raise ImportError() # do as there is no setuptools
+   from setuptools import setup
+   from setuptools.command import install_lib
+   USE_SETUPTOOLS = True
+except ImportError:
+   from distutils.core import setup
+   from distutils.command import install_lib
+   USE_SETUPTOOLS = False
 
 # import required features
-from __pkginfo__ import distname, version, license, short_desc, long_desc, \
-     web, author, author_email
+from __pkginfo__ import modname, version, license, description, web, \
+     author, author_email
+
+if exists('README'):
+   long_description = file('README').read()
+
 # import optional features
-try:
-    from __pkginfo__ import data_files
-except ImportError:
-    data_files = None
-try:
-    from __pkginfo__ import include_dirs
-except ImportError:
-    include_dirs = []
+import __pkginfo__
+if USE_SETUPTOOLS:
+   requires = {}
+   for entry in ("__depends__", "__recommends__"):
+      requires.update(getattr(__pkginfo__, entry, {}))
+   install_requires = [("%s %s" % (d, v and v or "")).strip()
+                       for d, v in requires.iteritems()]
+else:
+   install_requires = []
+
+distname = getattr(__pkginfo__, 'distname', modname)
+scripts = getattr(__pkginfo__, 'scripts', ())
+include_dirs = getattr(__pkginfo__, 'include_dirs', ())
+data_files = getattr(__pkginfo__, 'data_files', None)
+subpackage_of = getattr(__pkginfo__, 'subpackage_of', None)
+ext_modules = getattr(__pkginfo__, 'ext_modules', None)
+
+
+BASE_BLACKLIST = ('CVS', 'debian', 'dist', 'build', '__buildlog')
+IGNORED_EXTENSIONS = ('.pyc', '.pyo', '.elc')
+
+
+def ensure_scripts(linux_scripts):
+    """
+    Creates the proper script names required for each platform
+    (taken from 4Suite)
+    """
+    from distutils import util
+    if util.get_platform()[:3] == 'win':
+        scripts_ = [script + '.bat' for script in linux_scripts]
+    else:
+        scripts_ = linux_scripts
+    return scripts_
+
+
+def get_packages(directory, prefix):
+    """return a list of subpackages for the given directory
+    """
+    result = []
+    for package in os.listdir(directory):
+        absfile = join(directory, package)
+        if isdir(absfile):
+            if exists(join(absfile, '__init__.py')) or \
+                   package in ('test', 'tests'):
+                if prefix:
+                    result.append('%s.%s' % (prefix, package))
+                else:
+                    result.append(package)
+                result += get_packages(absfile, result[-1])
+    return result
+
+def export(from_dir, to_dir,
+           blacklist=BASE_BLACKLIST,
+           ignore_ext=IGNORED_EXTENSIONS,
+           verbose=True):
+    """make a mirror of from_dir in to_dir, omitting directories and files
+    listed in the black list
+    """
+    def make_mirror(arg, directory, fnames):
+        """walk handler"""
+        for norecurs in blacklist:
+            try:
+                fnames.remove(norecurs)
+            except ValueError:
+                pass
+        for filename in fnames:
+            # don't include binary files
+            if filename[-4:] in ignore_ext:
+                continue
+            if filename[-1] == '~':
+                continue
+            src = '%s/%s' % (directory, filename)
+            dest = to_dir + src[len(from_dir):]
+            if verbose:
+               print >> sys.stderr, src, '->', dest
+            if os.path.isdir(src):
+                if not exists(dest):
+                    os.mkdir(dest)
+            else:
+                if exists(dest):
+                    os.remove(dest)
+                shutil.copy2(src, dest)
+    try:
+        os.mkdir(to_dir)
+    except OSError, ex:
+        # file exists ?
+        import errno
+        if ex.errno != errno.EEXIST:
+            raise
+    walk(from_dir, make_mirror, None)
+
+
+EMPTY_FILE = '"""generated file, don\'t modify or your data will be lost"""\n'
+
+class MyInstallLib(install_lib.install_lib):
+    """extend install_lib command to handle  package __init__.py and
+    include_dirs variable if necessary
+    """
+    def run(self):
+        """overridden from install_lib class"""
+        install_lib.install_lib.run(self)
+        # create Products.__init__.py if needed
+        if subpackage_of:
+            product_init = join(self.install_dir, subpackage_of, '__init__.py')
+            if not exists(product_init):
+                self.announce('creating %s' % product_init)
+                stream = open(product_init, 'w')
+                stream.write(EMPTY_FILE)
+                stream.close()
+        # manually install included directories if any
+        if include_dirs:
+            if subpackage_of:
+                base = join(subpackage_of, modname)
+            else:
+                base = modname
+            for directory in include_dirs:
+                dest = join(self.install_dir, base, directory)
+                export(directory, dest, verbose=False)
 
 def install(**kwargs):
     """setup entry point"""
-    #kwargs['distname'] = modname
-    return setup(name=distname,
-                 version=version,
-                 license=license,
-                 description=short_desc,
-                 long_description=long_desc,
-                 author=author,
-                 author_email=author_email,
-                 url=web,
-                 data_files=data_files,
-                 **kwargs)
+    if not USE_SETUPTOOLS and '--install-layout=deb' in sys.argv and \
+           sys.versioninfo < (2, 5, 4):
+       sys.argv.remove('--install-layout=deb')
+       print "W: remove '--install-layout=deb' option"
+    if subpackage_of:
+        package = subpackage_of + '.' + modname
+        kwargs['package_dir'] = {package : '.'}
+        packages = [package] + get_packages(os.getcwd(), package)
+        if USE_SETUPTOOLS:
+            kwargs['namespace_packages'] = [subpackage_of]
+    else:
+        kwargs['package_dir'] = {modname : '.'}
+        packages = [modname] + get_packages(os.getcwd(), modname)
+    kwargs['packages'] = packages
+    return setup(name=distname, version=version, license=license, url=web,
+                 description=description, long_description=long_description,
+                 author=author, author_email=author_email,
+                 scripts=ensure_scripts(scripts), data_files=data_files,
+                 ext_modules=ext_modules,
+                 install_requires=install_requires,
+                 #dependency_links=["http://alain:alain@intranet.logilab.fr/~alain/"],
+                 cmdclass={'install_lib': MyInstallLib},
+                 **kwargs
+                 )
 
 if __name__ == '__main__' :
     install()
--- a/test/data/cubes/file/__pkginfo__.py	Thu Mar 25 20:32:52 2010 +0100
+++ b/test/data/cubes/file/__pkginfo__.py	Thu Mar 25 20:33:03 2010 +0100
@@ -13,48 +13,3 @@
 numversion = (1, 4, 3)
 version = '.'.join(str(num) for num in numversion)
 
-license = 'LGPL'
-copyright = '''Copyright (c) 2003-2010 LOGILAB S.A. (Paris, FRANCE).
-http://www.logilab.fr/ -- mailto:contact@logilab.fr'''
-
-author = "Logilab"
-author_email = "contact@logilab.fr"
-web = ''
-
-short_desc = "Raw file support for the CubicWeb framework"
-long_desc = """CubicWeb is a entities / relations bases knowledge management system
-developped at Logilab.
-.
-This package provides schema and views to store files and images in cubicweb
-applications.
-.
-"""
-
-from os import listdir
-from os.path import join
-
-CUBES_DIR = join('share', 'cubicweb', 'cubes')
-try:
-    data_files = [
-        [join(CUBES_DIR, 'file'),
-         [fname for fname in listdir('.')
-          if fname.endswith('.py') and fname != 'setup.py']],
-        [join(CUBES_DIR, 'file', 'data'),
-         [join('data', fname) for fname in listdir('data')]],
-        [join(CUBES_DIR, 'file', 'wdoc'),
-         [join('wdoc', fname) for fname in listdir('wdoc')]],
-        [join(CUBES_DIR, 'file', 'views'),
-         [join('views', fname) for fname in listdir('views') if fname.endswith('.py')]],
-        [join(CUBES_DIR, 'file', 'i18n'),
-         [join('i18n', fname) for fname in listdir('i18n')]],
-        [join(CUBES_DIR, 'file', 'migration'),
-         [join('migration', fname) for fname in listdir('migration')]],
-        ]
-except OSError:
-    # we are in an installed directory
-    pass
-
-
-cube_eid = 20320
-# used packages
-__use__ = ()