--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/cubicweb/cwconfig.py Sat Jan 16 13:48:51 2016 +0100
@@ -0,0 +1,1346 @@
+# -*- coding: utf-8 -*-
+# copyright 2003-2012 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of CubicWeb.
+#
+# CubicWeb is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option)
+# any later version.
+#
+# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
+"""
+.. _ResourceMode:
+
+Resource mode
+-------------
+
+Standard resource mode
+```````````````````````````
+
+A resource *mode* is a predefined set of settings for various resources
+directories, such as cubes, instances, etc. to ease development with the
+framework. There are two running modes with *CubicWeb*:
+
+* **system**: resources are searched / created in the system directories (eg
+ usually requiring root access):
+
+ - instances are stored in :file:`<INSTALL_PREFIX>/etc/cubicweb.d`
+ - temporary files (such as pid file) in :file:`<INSTALL_PREFIX>/var/run/cubicweb`
+
+ where `<INSTALL_PREFIX>` is the detected installation prefix ('/usr/local' for
+ instance).
+
+* **user**: resources are searched / created in the user home directory:
+
+ - instances are stored in :file:`~/etc/cubicweb.d`
+ - temporary files (such as pid file) in :file:`/tmp`
+
+
+
+
+.. _CubicwebWithinVirtualEnv:
+
+Within virtual environment
+```````````````````````````
+
+If you are not administrator of you machine or if you need to play with some
+specific version of |cubicweb| you can use `virtualenv`_ a tool to create
+isolated Python environments.
+
+- instances are stored in :file:`<VIRTUAL_ENV>/etc/cubicweb.d`
+- temporary files (such as pid file) in :file:`<VIRTUAL_ENV>/var/run/cubicweb`
+
+.. _`virtualenv`: http://pypi.python.org/pypi/virtualenv
+
+Custom resource location
+````````````````````````````````
+
+Notice that each resource path may be explicitly set using an environment
+variable if the default doesn't suit your needs. Here are the default resource
+directories that are affected according to mode:
+
+* **system**: ::
+
+ CW_INSTANCES_DIR = <INSTALL_PREFIX>/etc/cubicweb.d/
+ CW_INSTANCES_DATA_DIR = <INSTALL_PREFIX>/var/lib/cubicweb/instances/
+ CW_RUNTIME_DIR = <INSTALL_PREFIX>/var/run/cubicweb/
+
+* **user**: ::
+
+ CW_INSTANCES_DIR = ~/etc/cubicweb.d/
+ CW_INSTANCES_DATA_DIR = ~/etc/cubicweb.d/
+ CW_RUNTIME_DIR = /tmp
+
+Cubes search path is also affected, see the :ref:`Cube` section.
+
+Setting Cubicweb Mode
+`````````````````````
+
+By default, the mode is set to 'system' for standard installation. The mode is
+set to 'user' if `cubicweb is used from a mercurial repository`_. You can force
+this by setting the :envvar:`CW_MODE` environment variable to either 'user' or
+'system' so you can easily:
+
+* use system wide installation but user specific instances and all, without root
+ privileges on the system (`export CW_MODE=user`)
+
+* use local checkout of cubicweb on system wide instances (requires root
+ privileges on the system (`export CW_MODE=system`)
+
+If you've a doubt about the mode you're currently running, check the first line
+outputed by the :command:`cubicweb-ctl list` command.
+
+.. _`cubicweb is used from a mercurial repository`: CubicwebDevelopmentMod_
+
+.. _CubicwebDevelopmentMod:
+
+Development Mode
+`````````````````````
+If :file:`.hg` directory is found into the cubicweb package, there are specific resource rules.
+
+`<CW_SOFTWARE_ROOT>` is the source checkout's ``cubicweb`` directory:
+
+* main cubes directory is `<CW_SOFTWARE_ROOT>/../../cubes`. You can specify
+ another one with :envvar:`CW_INSTANCES_DIR` environment variable or simply
+ add some other directories by using :envvar:`CW_CUBES_PATH`
+
+* cubicweb migration files are searched in `<CW_SOFTWARE_ROOT>/misc/migration`
+ instead of `<INSTALL_PREFIX>/share/cubicweb/migration/`.
+
+
+.. _ConfigurationEnv:
+
+Environment configuration
+-------------------------
+
+Python
+``````
+
+If you installed *CubicWeb* by cloning the Mercurial shell repository or from source
+distribution, then you will need to update the environment variable PYTHONPATH by
+adding the path to `cubicweb`:
+
+Add the following lines to either :file:`.bashrc` or :file:`.bash_profile` to
+configure your development environment ::
+
+ export PYTHONPATH=/full/path/to/grshell-cubicweb
+
+If you installed *CubicWeb* with packages, no configuration is required and your
+new cubes will be placed in `/usr/share/cubicweb/cubes` and your instances will
+be placed in `/etc/cubicweb.d`.
+
+
+CubicWeb
+````````
+
+Here are all environment variables that may be used to configure *CubicWeb*:
+
+.. envvar:: CW_MODE
+
+ Resource mode: user or system, as explained in :ref:`ResourceMode`.
+
+.. envvar:: CW_CUBES_PATH
+
+ Augments the default search path for cubes. You may specify several
+ directories using ':' as separator (';' under windows environment).
+
+.. envvar:: CW_INSTANCES_DIR
+
+ Directory where cubicweb instances will be found.
+
+.. envvar:: CW_INSTANCES_DATA_DIR
+
+ Directory where cubicweb instances data will be written (backup file...)
+
+.. envvar:: CW_RUNTIME_DIR
+
+ Directory where pid files will be written
+"""
+from __future__ import print_function
+
+__docformat__ = "restructuredtext en"
+
+import sys
+import os
+import stat
+import logging
+import logging.config
+from smtplib import SMTP
+from threading import Lock
+from os.path import (exists, join, expanduser, abspath, normpath,
+ basename, isdir, dirname, splitext)
+from warnings import warn, filterwarnings
+
+from six import text_type
+
+from logilab.common.decorators import cached, classproperty
+from logilab.common.deprecation import deprecated
+from logilab.common.logging_ext import set_log_methods, init_log
+from logilab.common.configuration import (Configuration, Method,
+ ConfigurationMixIn, merge_options)
+
+from cubicweb import (CW_SOFTWARE_ROOT, CW_MIGRATION_MAP,
+ ConfigurationError, Binary, _)
+from cubicweb.toolsutils import create_dir
+
+CONFIGURATIONS = []
+
+SMTP_LOCK = Lock()
+
+
+def configuration_cls(name):
+ """return the configuration class registered with the given name"""
+ try:
+ return [c for c in CONFIGURATIONS if c.name == name][0]
+ except IndexError:
+ raise ConfigurationError('no such config %r (check it exists with "cubicweb-ctl list")' % name)
+
+def possible_configurations(directory):
+ """return a list of installed configurations in a directory
+ according to \*-ctl files
+ """
+ return [name for name in ('repository', 'all-in-one')
+ if exists(join(directory, '%s.conf' % name))]
+
+def guess_configuration(directory):
+ """try to guess the configuration to use for a directory. If multiple
+ configurations are found, ConfigurationError is raised
+ """
+ modes = possible_configurations(directory)
+ if len(modes) != 1:
+ raise ConfigurationError('unable to guess configuration from %r %s'
+ % (directory, modes))
+ return modes[0]
+
+def _find_prefix(start_path=CW_SOFTWARE_ROOT):
+ """Runs along the parent directories of *start_path* (default to cubicweb source directory)
+ looking for one containing a 'share/cubicweb' directory.
+ The first matching directory is assumed as the prefix installation of cubicweb
+
+ Returns the matching prefix or None.
+ """
+ prefix = start_path
+ old_prefix = None
+ if not isdir(start_path):
+ prefix = dirname(start_path)
+ while (not isdir(join(prefix, 'share', 'cubicweb'))
+ or prefix.endswith('.egg')) and prefix != old_prefix:
+ old_prefix = prefix
+ prefix = dirname(prefix)
+ if isdir(join(prefix, 'share', 'cubicweb')):
+ return prefix
+ return sys.prefix
+
+# persistent options definition
+PERSISTENT_OPTIONS = (
+ ('encoding',
+ {'type' : 'string',
+ 'default': 'UTF-8',
+ 'help': _('user interface encoding'),
+ 'group': 'ui', 'sitewide': True,
+ }),
+ ('language',
+ {'type' : 'string',
+ 'default': 'en',
+ 'vocabulary': Method('available_languages'),
+ 'help': _('language of the user interface'),
+ 'group': 'ui',
+ }),
+ ('date-format',
+ {'type' : 'string',
+ 'default': '%Y/%m/%d',
+ 'help': _('how to format date in the ui (see <a href="http://docs.python.org/library/datetime.html#strftime-strptime-behavior">this page</a> for format description)'),
+ 'group': 'ui',
+ }),
+ ('datetime-format',
+ {'type' : 'string',
+ 'default': '%Y/%m/%d %H:%M',
+ 'help': _('how to format date and time in the ui (see <a href="http://docs.python.org/library/datetime.html#strftime-strptime-behavior">this page</a> for format description)'),
+ 'group': 'ui',
+ }),
+ ('time-format',
+ {'type' : 'string',
+ 'default': '%H:%M',
+ 'help': _('how to format time in the ui (see <a href="http://docs.python.org/library/datetime.html#strftime-strptime-behavior">this page</a> for format description)'),
+ 'group': 'ui',
+ }),
+ ('float-format',
+ {'type' : 'string',
+ 'default': '%.3f',
+ 'help': _('how to format float numbers in the ui'),
+ 'group': 'ui',
+ }),
+ ('default-text-format',
+ {'type' : 'choice',
+ 'choices': ('text/plain', 'text/rest', 'text/html', 'text/markdown'),
+ 'default': 'text/plain',
+ 'help': _('default text format for rich text fields.'),
+ 'group': 'ui',
+ }),
+ ('short-line-size',
+ {'type' : 'int',
+ 'default': 80,
+ 'help': _('maximum number of characters in short description'),
+ 'group': 'navigation',
+ }),
+ )
+
+def register_persistent_options(options):
+ global PERSISTENT_OPTIONS
+ PERSISTENT_OPTIONS = merge_options(PERSISTENT_OPTIONS + options)
+
+CFGTYPE2ETYPE_MAP = {
+ 'string': 'String',
+ 'choice': 'String',
+ 'yn': 'Boolean',
+ 'int': 'Int',
+ 'float' : 'Float',
+ }
+
+_forced_mode = os.environ.get('CW_MODE')
+assert _forced_mode in (None, 'system', 'user')
+
+# CWDEV tells whether directories such as i18n/, web/data/, etc. (ie containing
+# some other resources than python libraries) are located with the python code
+# or as a 'shared' cube
+CWDEV = exists(join(CW_SOFTWARE_ROOT, 'i18n'))
+
+try:
+ _INSTALL_PREFIX = os.environ['CW_INSTALL_PREFIX']
+except KeyError:
+ _INSTALL_PREFIX = _find_prefix()
+_USR_INSTALL = _INSTALL_PREFIX == '/usr'
+
+class CubicWebNoAppConfiguration(ConfigurationMixIn):
+ """base class for cubicweb configuration without a specific instance directory
+ """
+ # to set in concrete configuration
+ name = None
+ # log messages format (see logging module documentation for available keys)
+ log_format = '%(asctime)s - (%(name)s) %(levelname)s: %(message)s'
+ # the format below can be useful to debug multi thread issues:
+ # log_format = '%(asctime)s - [%(threadName)s] (%(name)s) %(levelname)s: %(message)s'
+ # nor remove appobjects based on unused interface [???]
+ cleanup_unused_appobjects = True
+
+ quick_start = False
+
+ if (CWDEV and _forced_mode != 'system'):
+ mode = 'user'
+ _CUBES_DIR = join(CW_SOFTWARE_ROOT, '../../cubes')
+ else:
+ mode = _forced_mode or 'system'
+ _CUBES_DIR = join(_INSTALL_PREFIX, 'share', 'cubicweb', 'cubes')
+
+ CUBES_DIR = abspath(os.environ.get('CW_CUBES_DIR', _CUBES_DIR))
+ CUBES_PATH = os.environ.get('CW_CUBES_PATH', '').split(os.pathsep)
+
+ options = (
+ ('log-threshold',
+ {'type' : 'string', # XXX use a dedicated type?
+ 'default': 'WARNING',
+ 'help': 'server\'s log level',
+ 'group': 'main', 'level': 1,
+ }),
+ ('umask',
+ {'type' : 'int',
+ 'default': 0o077,
+ 'help': 'permission umask for files created by the server',
+ 'group': 'main', 'level': 2,
+ }),
+ # common configuration options which are potentially required as soon as
+ # you're using "base" application objects (ie to really server/web
+ # specific)
+ ('base-url',
+ {'type' : 'string',
+ 'default': None,
+ 'help': 'web server root url',
+ 'group': 'main', 'level': 1,
+ }),
+ ('allow-email-login',
+ {'type' : 'yn',
+ 'default': False,
+ 'help': 'allow users to login with their primary email if set',
+ 'group': 'main', 'level': 2,
+ }),
+ ('mangle-emails',
+ {'type' : 'yn',
+ 'default': False,
+ 'help': "don't display actual email addresses but mangle them if \
+this option is set to yes",
+ 'group': 'email', 'level': 3,
+ }),
+ )
+ # static and class methods used to get instance independant resources ##
+ @staticmethod
+ def cubicweb_version():
+ """return installed cubicweb version"""
+ from logilab.common.changelog import Version
+ from cubicweb import __pkginfo__
+ version = __pkginfo__.numversion
+ assert len(version) == 3, version
+ return Version(version)
+
+ @staticmethod
+ def persistent_options_configuration():
+ return Configuration(options=PERSISTENT_OPTIONS)
+
+ @classmethod
+ def shared_dir(cls):
+ """return the shared data directory (i.e. directory where standard
+ library views and data may be found)
+ """
+ if CWDEV:
+ return join(CW_SOFTWARE_ROOT, 'web')
+ return cls.cube_dir('shared')
+
+ @classmethod
+ def i18n_lib_dir(cls):
+ """return instance's i18n directory"""
+ if CWDEV:
+ return join(CW_SOFTWARE_ROOT, 'i18n')
+ return join(cls.shared_dir(), 'i18n')
+
+ @classmethod
+ def cw_languages(cls):
+ for fname in os.listdir(join(cls.i18n_lib_dir())):
+ if fname.endswith('.po'):
+ yield splitext(fname)[0]
+
+
+ @classmethod
+ def available_cubes(cls):
+ import re
+ cubes = set()
+ for directory in cls.cubes_search_path():
+ if not exists(directory):
+ cls.error('unexistant directory in cubes search path: %s'
+ % directory)
+ continue
+ for cube in os.listdir(directory):
+ if cube == 'shared':
+ continue
+ if not re.match('[_A-Za-z][_A-Za-z0-9]*$', cube):
+ continue # skip invalid python package name
+ cubedir = join(directory, cube)
+ if isdir(cubedir) and exists(join(cubedir, '__init__.py')):
+ cubes.add(cube)
+ return sorted(cubes)
+
+ @classmethod
+ def cubes_search_path(cls):
+ """return the path of directories where cubes should be searched"""
+ path = [abspath(normpath(directory)) for directory in cls.CUBES_PATH
+ if directory.strip() and exists(directory.strip())]
+ if not cls.CUBES_DIR in path and exists(cls.CUBES_DIR):
+ path.append(cls.CUBES_DIR)
+ return path
+
+ @classproperty
+ def extrapath(cls):
+ extrapath = {}
+ for cubesdir in cls.cubes_search_path():
+ if cubesdir != cls.CUBES_DIR:
+ extrapath[cubesdir] = 'cubes'
+ return extrapath
+
+ @classmethod
+ def cube_dir(cls, cube):
+ """return the cube directory for the given cube id, raise
+ `ConfigurationError` if it doesn't exist
+ """
+ for directory in cls.cubes_search_path():
+ cubedir = join(directory, cube)
+ if exists(cubedir):
+ return cubedir
+ raise ConfigurationError('no cube %r in %s' % (
+ cube, cls.cubes_search_path()))
+
+ @classmethod
+ def cube_migration_scripts_dir(cls, cube):
+ """cube migration scripts directory"""
+ return join(cls.cube_dir(cube), 'migration')
+
+ @classmethod
+ def cube_pkginfo(cls, cube):
+ """return the information module for the given cube"""
+ cube = CW_MIGRATION_MAP.get(cube, cube)
+ try:
+ parent = __import__('cubes.%s.__pkginfo__' % cube)
+ return getattr(parent, cube).__pkginfo__
+ except Exception as ex:
+ raise ConfigurationError(
+ 'unable to find packaging information for cube %s (%s: %s)'
+ % (cube, ex.__class__.__name__, ex))
+
+ @classmethod
+ def cube_version(cls, cube):
+ """return the version of the cube located in the given directory
+ """
+ from logilab.common.changelog import Version
+ version = cls.cube_pkginfo(cube).numversion
+ assert len(version) == 3, version
+ return Version(version)
+
+ @classmethod
+ def _cube_deps(cls, cube, key, oldkey):
+ """return cubicweb cubes used by the given cube"""
+ pkginfo = cls.cube_pkginfo(cube)
+ try:
+ # explicit __xxx_cubes__ attribute
+ deps = getattr(pkginfo, key)
+ except AttributeError:
+ # deduce cubes from generic __xxx__ attribute
+ try:
+ gendeps = getattr(pkginfo, key.replace('_cubes', ''))
+ except AttributeError:
+ deps = {}
+ else:
+ deps = dict( (x[len('cubicweb-'):], v)
+ for x, v in gendeps.items()
+ if x.startswith('cubicweb-'))
+ for depcube in deps:
+ try:
+ newname = CW_MIGRATION_MAP[depcube]
+ except KeyError:
+ pass
+ else:
+ deps[newname] = deps.pop(depcube)
+ return deps
+
+ @classmethod
+ def cube_depends_cubicweb_version(cls, cube):
+ # XXX no backward compat (see _cube_deps above)
+ try:
+ pkginfo = cls.cube_pkginfo(cube)
+ deps = getattr(pkginfo, '__depends__')
+ return deps.get('cubicweb')
+ except AttributeError:
+ return None
+
+ @classmethod
+ def cube_dependencies(cls, cube):
+ """return cubicweb cubes used by the given cube"""
+ return cls._cube_deps(cube, '__depends_cubes__', '__use__')
+
+ @classmethod
+ def cube_recommends(cls, cube):
+ """return cubicweb cubes recommended by the given cube"""
+ return cls._cube_deps(cube, '__recommends_cubes__', '__recommend__')
+
+ @classmethod
+ def expand_cubes(cls, cubes, with_recommends=False):
+ """expand the given list of top level cubes used by adding recursivly
+ each cube dependencies
+ """
+ cubes = list(cubes)
+ todo = cubes[:]
+ if with_recommends:
+ available = set(cls.available_cubes())
+ while todo:
+ cube = todo.pop(0)
+ for depcube in cls.cube_dependencies(cube):
+ if depcube not in cubes:
+ cubes.append(depcube)
+ todo.append(depcube)
+ if with_recommends:
+ for depcube in cls.cube_recommends(cube):
+ if depcube not in cubes and depcube in available:
+ cubes.append(depcube)
+ todo.append(depcube)
+ return cubes
+
+ @classmethod
+ def reorder_cubes(cls, cubes):
+ """reorder cubes from the top level cubes to inner dependencies
+ cubes
+ """
+ from logilab.common.graph import ordered_nodes, UnorderableGraph
+ graph = {}
+ for cube in cubes:
+ cube = CW_MIGRATION_MAP.get(cube, cube)
+ graph[cube] = set(dep for dep in cls.cube_dependencies(cube)
+ if dep in cubes)
+ graph[cube] |= set(dep for dep in cls.cube_recommends(cube)
+ if dep in cubes)
+ try:
+ return ordered_nodes(graph)
+ except UnorderableGraph as ex:
+ raise ConfigurationError(ex)
+
+ @classmethod
+ def cls_adjust_sys_path(cls):
+ """update python path if necessary"""
+ cubes_parent_dir = normpath(join(cls.CUBES_DIR, '..'))
+ if not cubes_parent_dir in sys.path:
+ sys.path.insert(0, cubes_parent_dir)
+ try:
+ import cubes
+ cubes.__path__ = cls.cubes_search_path()
+ except ImportError:
+ return # cubes dir doesn't exists
+
+ @classmethod
+ def load_available_configs(cls):
+ from logilab.common.modutils import load_module_from_file
+ for conffile in ('web/webconfig.py', 'etwist/twconfig.py',
+ 'server/serverconfig.py',):
+ if exists(join(CW_SOFTWARE_ROOT, conffile)):
+ load_module_from_file(join(CW_SOFTWARE_ROOT, conffile))
+
+ @classmethod
+ def load_cwctl_plugins(cls):
+ from logilab.common.modutils import load_module_from_file
+ cls.cls_adjust_sys_path()
+ for ctlfile in ('web/webctl.py', 'etwist/twctl.py',
+ 'server/serverctl.py',
+ 'devtools/devctl.py', 'goa/goactl.py'):
+ if exists(join(CW_SOFTWARE_ROOT, ctlfile)):
+ try:
+ load_module_from_file(join(CW_SOFTWARE_ROOT, ctlfile))
+ except ImportError as err:
+ cls.error('could not import the command provider %s: %s',
+ ctlfile, err)
+ cls.info('loaded cubicweb-ctl plugin %s', ctlfile)
+ for cube in cls.available_cubes():
+ pluginfile = join(cls.cube_dir(cube), 'ccplugin.py')
+ initfile = join(cls.cube_dir(cube), '__init__.py')
+ if exists(pluginfile):
+ try:
+ __import__('cubes.%s.ccplugin' % cube)
+ cls.info('loaded cubicweb-ctl plugin from %s', cube)
+ except Exception:
+ cls.exception('while loading plugin %s', pluginfile)
+ elif exists(initfile):
+ try:
+ __import__('cubes.%s' % cube)
+ except Exception:
+ cls.exception('while loading cube %s', cube)
+ else:
+ cls.warning('no __init__ file in cube %s', cube)
+
+ @classmethod
+ def init_available_cubes(cls):
+ """cubes may register some sources (svnfile for instance) in their
+ __init__ file, so they should be loaded early in the startup process
+ """
+ for cube in cls.available_cubes():
+ try:
+ __import__('cubes.%s' % cube)
+ except Exception as ex:
+ cls.warning("can't init cube %s: %s", cube, ex)
+
+ cubicweb_appobject_path = set(['entities'])
+ cube_appobject_path = set(['entities'])
+
+ def __init__(self, debugmode=False):
+ if debugmode:
+ # in python 2.7, DeprecationWarning are not shown anymore by default
+ filterwarnings('default', category=DeprecationWarning)
+ register_stored_procedures()
+ self._cubes = None
+ super(CubicWebNoAppConfiguration, self).__init__()
+ self.debugmode = debugmode
+ self.adjust_sys_path()
+ self.load_defaults()
+ # will be properly initialized later by _gettext_init
+ self.translations = {'en': (text_type, lambda ctx, msgid: text_type(msgid) )}
+ self._site_loaded = set()
+ # don't register ReStructured Text directives by simple import, avoid pb
+ # with eg sphinx.
+ # XXX should be done properly with a function from cw.uicfg
+ try:
+ from cubicweb.ext.rest import cw_rest_init
+ except ImportError:
+ pass
+ else:
+ cw_rest_init()
+
+ def adjust_sys_path(self):
+ # overriden in CubicWebConfiguration
+ self.cls_adjust_sys_path()
+
+ def init_log(self, logthreshold=None, logfile=None, syslog=False):
+ """init the log service"""
+ if logthreshold is None:
+ if self.debugmode:
+ logthreshold = 'DEBUG'
+ else:
+ logthreshold = self['log-threshold']
+ if sys.platform == 'win32':
+ # no logrotate on win32, so use logging rotation facilities
+ # for now, hard code weekly rotation every sunday, and 52 weeks kept
+ # idea: make this configurable?
+ init_log(self.debugmode, syslog, logthreshold, logfile, self.log_format,
+ rotation_parameters={'when': 'W6', # every sunday
+ 'interval': 1,
+ 'backupCount': 52})
+ else:
+ init_log(self.debugmode, syslog, logthreshold, logfile, self.log_format)
+ # configure simpleTal logger
+ logging.getLogger('simpleTAL').setLevel(logging.ERROR)
+
+ def appobjects_path(self):
+ """return a list of files or directories where the registry will look
+ for application objects. By default return nothing in NoApp config.
+ """
+ return []
+
+ def build_appobjects_path(self, templpath, evobjpath=None, tvobjpath=None):
+ """given a list of directories, return a list of sub files and
+ directories that should be loaded by the instance objects registry.
+
+ :param evobjpath:
+ optional list of sub-directories (or files without the .py ext) of
+ the cubicweb library that should be tested and added to the output list
+ if they exists. If not give, default to `cubicweb_appobject_path` class
+ attribute.
+ :param tvobjpath:
+ optional list of sub-directories (or files without the .py ext) of
+ directories given in `templpath` that should be tested and added to
+ the output list if they exists. If not give, default to
+ `cube_appobject_path` class attribute.
+ """
+ vregpath = self.build_appobjects_cubicweb_path(evobjpath)
+ vregpath += self.build_appobjects_cube_path(templpath, tvobjpath)
+ return vregpath
+
+ def build_appobjects_cubicweb_path(self, evobjpath=None):
+ vregpath = []
+ if evobjpath is None:
+ evobjpath = self.cubicweb_appobject_path
+ # NOTE: for the order, see http://www.cubicweb.org/ticket/2330799
+ # it is clearly a workaround
+ for subdir in sorted(evobjpath, key=lambda x:x != 'entities'):
+ path = join(CW_SOFTWARE_ROOT, subdir)
+ if exists(path):
+ vregpath.append(path)
+ return vregpath
+
+ def build_appobjects_cube_path(self, templpath, tvobjpath=None):
+ vregpath = []
+ if tvobjpath is None:
+ tvobjpath = self.cube_appobject_path
+ for directory in templpath:
+ # NOTE: for the order, see http://www.cubicweb.org/ticket/2330799
+ for subdir in sorted(tvobjpath, key=lambda x:x != 'entities'):
+ path = join(directory, subdir)
+ if exists(path):
+ vregpath.append(path)
+ elif exists(path + '.py'):
+ vregpath.append(path + '.py')
+ return vregpath
+
+ apphome = None
+
+ def load_site_cubicweb(self, paths=None):
+ """load instance's specific site_cubicweb file"""
+ if paths is None:
+ paths = self.cubes_path()
+ if self.apphome is not None:
+ paths = [self.apphome] + paths
+ for path in reversed(paths):
+ sitefile = join(path, 'site_cubicweb.py')
+ if exists(sitefile) and not sitefile in self._site_loaded:
+ self._load_site_cubicweb(sitefile)
+ self._site_loaded.add(sitefile)
+
+ def _load_site_cubicweb(self, sitefile):
+ # XXX extrapath argument to load_module_from_file only in lgc > 0.50.2
+ from logilab.common.modutils import load_module_from_modpath, modpath_from_file
+ module = load_module_from_modpath(modpath_from_file(sitefile, self.extrapath))
+ self.debug('%s loaded', sitefile)
+ return module
+
+ def cwproperty_definitions(self):
+ cfg = self.persistent_options_configuration()
+ for section, options in cfg.options_by_section():
+ section = section.lower()
+ for optname, optdict, value in options:
+ key = '%s.%s' % (section, optname)
+ type, vocab = self.map_option(optdict)
+ default = cfg.option_default(optname, optdict)
+ pdef = {'type': type, 'vocabulary': vocab, 'default': default,
+ 'help': optdict['help'],
+ 'sitewide': optdict.get('sitewide', False)}
+ yield key, pdef
+
+ def map_option(self, optdict):
+ try:
+ vocab = optdict['choices']
+ except KeyError:
+ vocab = optdict.get('vocabulary')
+ if isinstance(vocab, Method):
+ vocab = getattr(self, vocab.method, ())
+ return CFGTYPE2ETYPE_MAP[optdict['type']], vocab
+
+ def default_instance_id(self):
+ """return the instance identifier, useful for option which need this
+ as default value
+ """
+ return None
+
+ _cubes = None
+
+ def init_cubes(self, cubes):
+ self._cubes = self.reorder_cubes(cubes)
+ # load cubes'__init__.py file first
+ for cube in cubes:
+ __import__('cubes.%s' % cube)
+ self.load_site_cubicweb()
+
+ def cubes(self):
+ """return the list of cubes used by this instance
+
+ result is ordered from the top level cubes to inner dependencies
+ cubes
+ """
+ assert self._cubes is not None, 'cubes not initialized'
+ return self._cubes
+
+ def cubes_path(self):
+ """return the list of path to cubes used by this instance, from outer
+ most to inner most cubes
+ """
+ return [self.cube_dir(p) for p in self.cubes()]
+
+ # these are overridden by set_log_methods below
+ # only defining here to prevent pylint from complaining
+ @classmethod
+ def debug(cls, msg, *a, **kw):
+ pass
+ info = warning = error = critical = exception = debug
+
+
+class CubicWebConfiguration(CubicWebNoAppConfiguration):
+ """base class for cubicweb server and web configurations"""
+
+ if CubicWebNoAppConfiguration.mode == 'user':
+ _INSTANCES_DIR = expanduser('~/etc/cubicweb.d/')
+ #mode == system'
+ elif _USR_INSTALL:
+ _INSTANCES_DIR = '/etc/cubicweb.d/'
+ else:
+ _INSTANCES_DIR = join(_INSTALL_PREFIX, 'etc', 'cubicweb.d')
+
+ # set to true during repair (shell, migration) to allow some things which
+ # wouldn't be possible otherwise
+ repairing = False
+
+ # set by upgrade command
+ verbosity = 0
+ cmdline_options = None
+ options = CubicWebNoAppConfiguration.options + (
+ ('log-file',
+ {'type' : 'string',
+ 'default': Method('default_log_file'),
+ 'help': 'file where output logs should be written',
+ 'group': 'main', 'level': 2,
+ }),
+ ('statsd-endpoint',
+ {'type' : 'string',
+ 'default': '',
+ 'help': 'UDP address of the statsd endpoint; it must be formatted'
+ 'like <ip>:<port>; disabled is unset.',
+ 'group': 'main', 'level': 2,
+ }),
+ # email configuration
+ ('smtp-host',
+ {'type' : 'string',
+ 'default': 'mail',
+ 'help': 'hostname of the SMTP mail server',
+ 'group': 'email', 'level': 1,
+ }),
+ ('smtp-port',
+ {'type' : 'int',
+ 'default': 25,
+ 'help': 'listening port of the SMTP mail server',
+ 'group': 'email', 'level': 1,
+ }),
+ ('sender-name',
+ {'type' : 'string',
+ 'default': Method('default_instance_id'),
+ 'help': 'name used as HELO name for outgoing emails from the \
+repository.',
+ 'group': 'email', 'level': 2,
+ }),
+ ('sender-addr',
+ {'type' : 'string',
+ 'default': 'cubicweb@mydomain.com',
+ 'help': 'email address used as HELO address for outgoing emails from \
+the repository',
+ 'group': 'email', 'level': 1,
+ }),
+ ('logstat-interval',
+ {'type' : 'int',
+ 'default': 0,
+ 'help': 'interval (in seconds) at which stats are dumped in the logstat file; set 0 to disable',
+ 'group': 'main', 'level': 2,
+ }),
+ ('logstat-file',
+ {'type' : 'string',
+ 'default': Method('default_stats_file'),
+ 'help': 'file where stats for the instance should be written',
+ 'group': 'main', 'level': 2,
+ }),
+ )
+
+ @classmethod
+ def instances_dir(cls):
+ """return the control directory"""
+ return abspath(os.environ.get('CW_INSTANCES_DIR', cls._INSTANCES_DIR))
+
+ @classmethod
+ def migration_scripts_dir(cls):
+ """cubicweb migration scripts directory"""
+ if CWDEV:
+ return join(CW_SOFTWARE_ROOT, 'misc', 'migration')
+ mdir = join(_INSTALL_PREFIX, 'share', 'cubicweb', 'migration')
+ if not exists(mdir):
+ raise ConfigurationError('migration path %s doesn\'t exist' % mdir)
+ return mdir
+
+ @classmethod
+ def config_for(cls, appid, config=None, debugmode=False, creating=False):
+ """return a configuration instance for the given instance identifier
+ """
+ cls.load_available_configs()
+ config = config or guess_configuration(cls.instance_home(appid))
+ configcls = configuration_cls(config)
+ return configcls(appid, debugmode, creating)
+
+ @classmethod
+ def possible_configurations(cls, appid):
+ """return the name of possible configurations for the given
+ instance id
+ """
+ home = cls.instance_home(appid)
+ return possible_configurations(home)
+
+ @classmethod
+ def instance_home(cls, appid):
+ """return the home directory of the instance with the given
+ instance id
+ """
+ home = join(cls.instances_dir(), appid)
+ if not exists(home):
+ raise ConfigurationError('no such instance %s (check it exists with'
+ ' "cubicweb-ctl list")' % appid)
+ return home
+
+ MODES = ('common', 'repository', 'Any')
+ MCOMPAT = {'all-in-one': MODES,
+ 'repository': ('common', 'repository', 'Any')}
+ @classmethod
+ def accept_mode(cls, mode):
+ #assert mode in cls.MODES, mode
+ return mode in cls.MCOMPAT[cls.name]
+
+ # default configuration methods ###########################################
+
+ def default_instance_id(self):
+ """return the instance identifier, useful for option which need this
+ as default value
+ """
+ return self.appid
+
+ def default_log_file(self):
+ """return default path to the log file of the instance'server"""
+ if self.mode == 'user':
+ import tempfile
+ basepath = join(tempfile.gettempdir(), '%s-%s' % (
+ basename(self.appid), self.name))
+ path = basepath + '.log'
+ i = 1
+ while exists(path) and i < 100: # arbitrary limit to avoid infinite loop
+ try:
+ open(path, 'a')
+ break
+ except IOError:
+ path = '%s-%s.log' % (basepath, i)
+ i += 1
+ return path
+ if _USR_INSTALL:
+ return '/var/log/cubicweb/%s-%s.log' % (self.appid, self.name)
+ else:
+ log_path = os.path.join(_INSTALL_PREFIX, 'var', 'log', 'cubicweb', '%s-%s.log')
+ return log_path % (self.appid, self.name)
+
+ def default_stats_file(self):
+ """return default path to the stats file of the instance'server"""
+ logfile = self.default_log_file()
+ if logfile.endswith('.log'):
+ logfile = logfile[:-4]
+ return logfile + '.stats'
+
+ def default_pid_file(self):
+ """return default path to the pid file of the instance'server"""
+ if self.mode == 'system':
+ if _USR_INSTALL:
+ default = '/var/run/cubicweb/'
+ else:
+ default = os.path.join(_INSTALL_PREFIX, 'var', 'run', 'cubicweb')
+ else:
+ import tempfile
+ default = tempfile.gettempdir()
+ # runtime directory created on startup if necessary, don't check it
+ # exists
+ rtdir = abspath(os.environ.get('CW_RUNTIME_DIR', default))
+ return join(rtdir, '%s-%s.pid' % (self.appid, self.name))
+
+ # config -> repository
+
+ def repository(self, vreg=None):
+ from cubicweb.server.repository import Repository
+ from cubicweb.server.utils import TasksManager
+ return Repository(self, TasksManager(), vreg=vreg)
+
+ # instance methods used to get instance specific resources #############
+
+ def __init__(self, appid, debugmode=False, creating=False):
+ self.appid = appid
+ # set to true while creating an instance
+ self.creating = creating
+ super(CubicWebConfiguration, self).__init__(debugmode)
+ fake_gettext = (text_type, lambda ctx, msgid: text_type(msgid))
+ for lang in self.available_languages():
+ self.translations[lang] = fake_gettext
+ self._cubes = None
+ self.load_file_configuration(self.main_config_file())
+
+ def adjust_sys_path(self):
+ super(CubicWebConfiguration, self).adjust_sys_path()
+ # adding apphome to python path is not usually necessary in production
+ # environments, but necessary for tests
+ if self.apphome and self.apphome not in sys.path:
+ sys.path.insert(0, self.apphome)
+
+ @property
+ def apphome(self):
+ return join(self.instances_dir(), self.appid)
+
+ @property
+ def appdatahome(self):
+ if self.mode == 'system':
+ if _USR_INSTALL:
+ iddir = os.path.join('/var','lib', 'cubicweb', 'instances')
+ else:
+ iddir = os.path.join(_INSTALL_PREFIX, 'var', 'lib', 'cubicweb', 'instances')
+ else:
+ iddir = self.instances_dir()
+ iddir = abspath(os.environ.get('CW_INSTANCES_DATA_DIR', iddir))
+ return join(iddir, self.appid)
+
+ def init_cubes(self, cubes):
+ super(CubicWebConfiguration, self).init_cubes(cubes)
+ # reload config file in cases options are defined in cubes __init__
+ # or site_cubicweb files
+ self.load_file_configuration(self.main_config_file())
+ # configuration initialization hook
+ self.load_configuration(**(self.cmdline_options or {}))
+
+ def add_cubes(self, cubes):
+ """add given cubes to the list of used cubes"""
+ if not isinstance(cubes, list):
+ cubes = list(cubes)
+ self._cubes = self.reorder_cubes(list(self._cubes) + cubes)
+ self.load_site_cubicweb([self.cube_dir(cube) for cube in cubes])
+
+ def main_config_file(self):
+ """return instance's control configuration file"""
+ return join(self.apphome, '%s.conf' % self.name)
+
+ def save(self):
+ """write down current configuration"""
+ with open(self.main_config_file(), 'w') as fobj:
+ self.generate_config(fobj)
+
+ def check_writeable_uid_directory(self, path):
+ """check given directory path exists, belongs to the user running the
+ server process and is writeable.
+
+ If not, try to fix this, letting exception propagate when not possible.
+ """
+ if not exists(path):
+ self.info('creating %s directory', path)
+ try:
+ os.makedirs(path)
+ except OSError as ex:
+ self.warning('error while creating %s directory: %s', path, ex)
+ return
+ if self['uid']:
+ try:
+ uid = int(self['uid'])
+ except ValueError:
+ from pwd import getpwnam
+ uid = getpwnam(self['uid']).pw_uid
+ else:
+ try:
+ uid = os.getuid()
+ except AttributeError: # we are on windows
+ return
+ fstat = os.stat(path)
+ if fstat.st_uid != uid:
+ self.info('giving ownership of %s directory to %s', path, self['uid'])
+ try:
+ os.chown(path, uid, os.getgid())
+ except OSError as ex:
+ self.warning('error while giving ownership of %s directory to %s: %s',
+ path, self['uid'], ex)
+ if not (fstat.st_mode & stat.S_IWUSR):
+ self.info('forcing write permission on directory %s', path)
+ try:
+ os.chmod(path, fstat.st_mode | stat.S_IWUSR)
+ except OSError as ex:
+ self.warning('error while forcing write permission on directory %s: %s',
+ path, ex)
+ return
+
+ @cached
+ def instance_md5_version(self):
+ from hashlib import md5 # pylint: disable=E0611
+ infos = []
+ for pkg in sorted(self.cubes()):
+ version = self.cube_version(pkg)
+ infos.append('%s-%s' % (pkg, version))
+ infos.append('cubicweb-%s' % str(self.cubicweb_version()))
+ return md5((';'.join(infos)).encode('ascii')).hexdigest()
+
+ def load_configuration(self, **kw):
+ """load instance's configuration files"""
+ super(CubicWebConfiguration, self).load_configuration(**kw)
+ if self.apphome and not self.creating:
+ # init gettext
+ self._gettext_init()
+
+ def _load_site_cubicweb(self, sitefile):
+ # overridden to register cube specific options
+ mod = super(CubicWebConfiguration, self)._load_site_cubicweb(sitefile)
+ if getattr(mod, 'options', None):
+ self.register_options(mod.options)
+ self.load_defaults()
+
+ def init_log(self, logthreshold=None, force=False):
+ """init the log service"""
+ if not force and hasattr(self, '_logging_initialized'):
+ return
+ self._logging_initialized = True
+ super_self = super(CubicWebConfiguration, self)
+ super_self.init_log(logthreshold, logfile=self.get('log-file'))
+ # read a config file if it exists
+ logconfig = join(self.apphome, 'logging.conf')
+ if exists(logconfig):
+ logging.config.fileConfig(logconfig)
+ # set the statsd address, if any
+ if self.get('statsd-endpoint'):
+ try:
+ address, port = self.get('statsd-endpoint').split(':')
+ port = int(port)
+ except:
+ self.error('statsd-endpoint: invalid address format ({}); '
+ 'it should be "ip:port"'.format(self.get('statsd-endpoint')))
+ else:
+ import statsd_logger
+ statsd_logger.setup('cubicweb.%s' % self.appid, (address, port))
+
+ def available_languages(self, *args):
+ """return available translation for an instance, by looking for
+ compiled catalog
+
+ take \*args to be usable as a vocabulary method
+ """
+ from glob import glob
+ yield 'en' # ensure 'en' is yielded even if no .mo found
+ for path in glob(join(self.apphome, 'i18n',
+ '*', 'LC_MESSAGES')):
+ lang = path.split(os.sep)[-2]
+ if lang != 'en':
+ yield lang
+
+ def _gettext_init(self):
+ """set language for gettext"""
+ from cubicweb.cwgettext import translation
+ path = join(self.apphome, 'i18n')
+ for language in self.available_languages():
+ self.info("loading language %s", language)
+ try:
+ tr = translation('cubicweb', path, languages=[language])
+ self.translations[language] = (tr.ugettext, tr.upgettext)
+ except (ImportError, AttributeError, IOError):
+ if self.mode != 'test':
+ # in test contexts, data/i18n does not exist, hence
+ # logging will only pollute the logs
+ self.exception('localisation support error for language %s',
+ language)
+
+ def appobjects_path(self):
+ """return a list of files or directories where the registry will look
+ for application objects
+ """
+ templpath = list(reversed(self.cubes_path()))
+ if self.apphome: # may be unset in tests
+ templpath.append(self.apphome)
+ return self.build_appobjects_path(templpath)
+
+ def set_sources_mode(self, sources):
+ if not 'all' in sources:
+ print('warning: ignoring specified sources, requires a repository '
+ 'configuration')
+
+ def i18ncompile(self, langs=None):
+ from cubicweb import i18n
+ if langs is None:
+ langs = self.available_languages()
+ i18ndir = join(self.apphome, 'i18n')
+ if not exists(i18ndir):
+ create_dir(i18ndir)
+ sourcedirs = [join(path, 'i18n') for path in self.cubes_path()]
+ sourcedirs.append(self.i18n_lib_dir())
+ return i18n.compile_i18n_catalogs(sourcedirs, i18ndir, langs)
+
+ def sendmails(self, msgs, fromaddr=None):
+ """msgs: list of 2-uple (message object, recipients). Return False
+ if connection to the smtp server failed, else True.
+ """
+ server, port = self['smtp-host'], self['smtp-port']
+ if fromaddr is None:
+ fromaddr = '%s <%s>' % (self['sender-name'], self['sender-addr'])
+ SMTP_LOCK.acquire()
+ try:
+ try:
+ smtp = SMTP(server, port)
+ except Exception as ex:
+ self.exception("can't connect to smtp server %s:%s (%s)",
+ server, port, ex)
+ return False
+ for msg, recipients in msgs:
+ try:
+ smtp.sendmail(fromaddr, recipients, msg.as_string())
+ except Exception as ex:
+ self.exception("error sending mail to %s (%s)",
+ recipients, ex)
+ smtp.close()
+ finally:
+ SMTP_LOCK.release()
+ return True
+
+set_log_methods(CubicWebNoAppConfiguration,
+ logging.getLogger('cubicweb.configuration'))
+
+# alias to get a configuration instance from an instance id
+instance_configuration = CubicWebConfiguration.config_for
+application_configuration = deprecated('use instance_configuration')(instance_configuration)
+
+
+_EXT_REGISTERED = False
+def register_stored_procedures():
+ from logilab.database import FunctionDescr
+ from rql.utils import register_function, iter_funcnode_variables
+ from rql.nodes import SortTerm, Constant, VariableRef
+
+ global _EXT_REGISTERED
+ if _EXT_REGISTERED:
+ return
+ _EXT_REGISTERED = True
+
+ class COMMA_JOIN(FunctionDescr):
+ supported_backends = ('postgres', 'sqlite',)
+ rtype = 'String'
+
+ def st_description(self, funcnode, mainindex, tr):
+ return ', '.join(sorted(term.get_description(mainindex, tr)
+ for term in iter_funcnode_variables(funcnode)))
+
+ register_function(COMMA_JOIN) # XXX do not expose?
+
+
+ class CONCAT_STRINGS(COMMA_JOIN):
+ aggregat = True
+
+ register_function(CONCAT_STRINGS) # XXX bw compat
+
+
+ class GROUP_CONCAT(CONCAT_STRINGS):
+ supported_backends = ('mysql', 'postgres', 'sqlite',)
+
+ register_function(GROUP_CONCAT)
+
+
+ class LIMIT_SIZE(FunctionDescr):
+ supported_backends = ('postgres', 'sqlite',)
+ minargs = maxargs = 3
+ rtype = 'String'
+
+ def st_description(self, funcnode, mainindex, tr):
+ return funcnode.children[0].get_description(mainindex, tr)
+
+ register_function(LIMIT_SIZE)
+
+
+ class TEXT_LIMIT_SIZE(LIMIT_SIZE):
+ supported_backends = ('mysql', 'postgres', 'sqlite',)
+ minargs = maxargs = 2
+
+ register_function(TEXT_LIMIT_SIZE)
+
+
+ class FTIRANK(FunctionDescr):
+ """return ranking of a variable that must be used as some has_text
+ relation subject in the query's restriction. Usually used to sort result
+ of full-text search by ranking.
+ """
+ supported_backends = ('postgres',)
+ rtype = 'Float'
+
+ def st_check_backend(self, backend, funcnode):
+ """overriden so that on backend not supporting fti ranking, the
+ function is removed when in an orderby clause, or replaced by a 1.0
+ constant.
+ """
+ if not self.supports(backend):
+ parent = funcnode.parent
+ while parent is not None and not isinstance(parent, SortTerm):
+ parent = parent.parent
+ if isinstance(parent, SortTerm):
+ parent.parent.remove(parent)
+ else:
+ funcnode.parent.replace(funcnode, Constant(1.0, 'Float'))
+ parent = funcnode
+ for vref in parent.iget_nodes(VariableRef):
+ vref.unregister_reference()
+
+ register_function(FTIRANK)
+
+
+ class FSPATH(FunctionDescr):
+ """return path of some bytes attribute stored using the Bytes
+ File-System Storage (bfss)
+ """
+ rtype = 'Bytes' # XXX return a String? potential pb with fs encoding
+
+ def update_cb_stack(self, stack):
+ assert len(stack) == 1
+ stack[0] = self.source_execute
+
+ def as_sql(self, backend, args):
+ raise NotImplementedError(
+ 'This callback is only available for BytesFileSystemStorage '
+ 'managed attribute. Is FSPATH() argument BFSS managed?')
+
+ def source_execute(self, source, session, value):
+ fpath = source.binary_to_str(value)
+ try:
+ return Binary(fpath)
+ except OSError as ex:
+ source.critical("can't open %s: %s", fpath, ex)
+ return None
+
+ register_function(FSPATH)