diff -r 058bb3dc685f -r 0b59724cb3f2 cwconfig.py --- a/cwconfig.py Mon Jan 04 18:40:30 2016 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1346 +0,0 @@ -# -*- coding: utf-8 -*- -# copyright 2003-2012 LOGILAB S.A. (Paris, FRANCE), all rights reserved. -# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr -# -# This file is part of CubicWeb. -# -# CubicWeb is free software: you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation, either version 2.1 of the License, or (at your option) -# any later version. -# -# CubicWeb is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License along -# with CubicWeb. If not, see . -""" -.. _ResourceMode: - -Resource mode -------------- - -Standard resource mode -``````````````````````````` - -A resource *mode* is a predefined set of settings for various resources -directories, such as cubes, instances, etc. to ease development with the -framework. There are two running modes with *CubicWeb*: - -* **system**: resources are searched / created in the system directories (eg - usually requiring root access): - - - instances are stored in :file:`/etc/cubicweb.d` - - temporary files (such as pid file) in :file:`/var/run/cubicweb` - - where `` is the detected installation prefix ('/usr/local' for - instance). - -* **user**: resources are searched / created in the user home directory: - - - instances are stored in :file:`~/etc/cubicweb.d` - - temporary files (such as pid file) in :file:`/tmp` - - - - -.. _CubicwebWithinVirtualEnv: - -Within virtual environment -``````````````````````````` - -If you are not administrator of you machine or if you need to play with some -specific version of |cubicweb| you can use `virtualenv`_ a tool to create -isolated Python environments. - -- instances are stored in :file:`/etc/cubicweb.d` -- temporary files (such as pid file) in :file:`/var/run/cubicweb` - -.. _`virtualenv`: http://pypi.python.org/pypi/virtualenv - -Custom resource location -```````````````````````````````` - -Notice that each resource path may be explicitly set using an environment -variable if the default doesn't suit your needs. Here are the default resource -directories that are affected according to mode: - -* **system**: :: - - CW_INSTANCES_DIR = /etc/cubicweb.d/ - CW_INSTANCES_DATA_DIR = /var/lib/cubicweb/instances/ - CW_RUNTIME_DIR = /var/run/cubicweb/ - -* **user**: :: - - CW_INSTANCES_DIR = ~/etc/cubicweb.d/ - CW_INSTANCES_DATA_DIR = ~/etc/cubicweb.d/ - CW_RUNTIME_DIR = /tmp - -Cubes search path is also affected, see the :ref:`Cube` section. - -Setting Cubicweb Mode -````````````````````` - -By default, the mode is set to 'system' for standard installation. The mode is -set to 'user' if `cubicweb is used from a mercurial repository`_. You can force -this by setting the :envvar:`CW_MODE` environment variable to either 'user' or -'system' so you can easily: - -* use system wide installation but user specific instances and all, without root - privileges on the system (`export CW_MODE=user`) - -* use local checkout of cubicweb on system wide instances (requires root - privileges on the system (`export CW_MODE=system`) - -If you've a doubt about the mode you're currently running, check the first line -outputed by the :command:`cubicweb-ctl list` command. - -.. _`cubicweb is used from a mercurial repository`: CubicwebDevelopmentMod_ - -.. _CubicwebDevelopmentMod: - -Development Mode -````````````````````` -If :file:`.hg` directory is found into the cubicweb package, there are specific resource rules. - -`` is the mercurial checkout of cubicweb: - -* main cubes directory is `/../cubes`. You can specify - another one with :envvar:`CW_INSTANCES_DIR` environment variable or simply - add some other directories by using :envvar:`CW_CUBES_PATH` - -* cubicweb migration files are searched in `/misc/migration` - instead of `/share/cubicweb/migration/`. - - -.. _ConfigurationEnv: - -Environment configuration -------------------------- - -Python -`````` - -If you installed *CubicWeb* by cloning the Mercurial shell repository or from source -distribution, then you will need to update the environment variable PYTHONPATH by -adding the path to `cubicweb`: - -Add the following lines to either :file:`.bashrc` or :file:`.bash_profile` to -configure your development environment :: - - export PYTHONPATH=/full/path/to/grshell-cubicweb - -If you installed *CubicWeb* with packages, no configuration is required and your -new cubes will be placed in `/usr/share/cubicweb/cubes` and your instances will -be placed in `/etc/cubicweb.d`. - - -CubicWeb -```````` - -Here are all environment variables that may be used to configure *CubicWeb*: - -.. envvar:: CW_MODE - - Resource mode: user or system, as explained in :ref:`ResourceMode`. - -.. envvar:: CW_CUBES_PATH - - Augments the default search path for cubes. You may specify several - directories using ':' as separator (';' under windows environment). - -.. envvar:: CW_INSTANCES_DIR - - Directory where cubicweb instances will be found. - -.. envvar:: CW_INSTANCES_DATA_DIR - - Directory where cubicweb instances data will be written (backup file...) - -.. envvar:: CW_RUNTIME_DIR - - Directory where pid files will be written -""" -from __future__ import print_function - -__docformat__ = "restructuredtext en" - -import sys -import os -import stat -import logging -import logging.config -from smtplib import SMTP -from threading import Lock -from os.path import (exists, join, expanduser, abspath, normpath, - basename, isdir, dirname, splitext) -from warnings import warn, filterwarnings - -from six import text_type - -from logilab.common.decorators import cached, classproperty -from logilab.common.deprecation import deprecated -from logilab.common.logging_ext import set_log_methods, init_log -from logilab.common.configuration import (Configuration, Method, - ConfigurationMixIn, merge_options) - -from cubicweb import (CW_SOFTWARE_ROOT, CW_MIGRATION_MAP, - ConfigurationError, Binary, _) -from cubicweb.toolsutils import create_dir - -CONFIGURATIONS = [] - -SMTP_LOCK = Lock() - - -def configuration_cls(name): - """return the configuration class registered with the given name""" - try: - return [c for c in CONFIGURATIONS if c.name == name][0] - except IndexError: - raise ConfigurationError('no such config %r (check it exists with "cubicweb-ctl list")' % name) - -def possible_configurations(directory): - """return a list of installed configurations in a directory - according to \*-ctl files - """ - return [name for name in ('repository', 'all-in-one') - if exists(join(directory, '%s.conf' % name))] - -def guess_configuration(directory): - """try to guess the configuration to use for a directory. If multiple - configurations are found, ConfigurationError is raised - """ - modes = possible_configurations(directory) - if len(modes) != 1: - raise ConfigurationError('unable to guess configuration from %r %s' - % (directory, modes)) - return modes[0] - -def _find_prefix(start_path=CW_SOFTWARE_ROOT): - """Runs along the parent directories of *start_path* (default to cubicweb source directory) - looking for one containing a 'share/cubicweb' directory. - The first matching directory is assumed as the prefix installation of cubicweb - - Returns the matching prefix or None. - """ - prefix = start_path - old_prefix = None - if not isdir(start_path): - prefix = dirname(start_path) - while (not isdir(join(prefix, 'share', 'cubicweb')) - or prefix.endswith('.egg')) and prefix != old_prefix: - old_prefix = prefix - prefix = dirname(prefix) - if isdir(join(prefix, 'share', 'cubicweb')): - return prefix - return sys.prefix - -# persistent options definition -PERSISTENT_OPTIONS = ( - ('encoding', - {'type' : 'string', - 'default': 'UTF-8', - 'help': _('user interface encoding'), - 'group': 'ui', 'sitewide': True, - }), - ('language', - {'type' : 'string', - 'default': 'en', - 'vocabulary': Method('available_languages'), - 'help': _('language of the user interface'), - 'group': 'ui', - }), - ('date-format', - {'type' : 'string', - 'default': '%Y/%m/%d', - 'help': _('how to format date in the ui (see this page for format description)'), - 'group': 'ui', - }), - ('datetime-format', - {'type' : 'string', - 'default': '%Y/%m/%d %H:%M', - 'help': _('how to format date and time in the ui (see this page for format description)'), - 'group': 'ui', - }), - ('time-format', - {'type' : 'string', - 'default': '%H:%M', - 'help': _('how to format time in the ui (see this page for format description)'), - 'group': 'ui', - }), - ('float-format', - {'type' : 'string', - 'default': '%.3f', - 'help': _('how to format float numbers in the ui'), - 'group': 'ui', - }), - ('default-text-format', - {'type' : 'choice', - 'choices': ('text/plain', 'text/rest', 'text/html', 'text/markdown'), - 'default': 'text/plain', - 'help': _('default text format for rich text fields.'), - 'group': 'ui', - }), - ('short-line-size', - {'type' : 'int', - 'default': 80, - 'help': _('maximum number of characters in short description'), - 'group': 'navigation', - }), - ) - -def register_persistent_options(options): - global PERSISTENT_OPTIONS - PERSISTENT_OPTIONS = merge_options(PERSISTENT_OPTIONS + options) - -CFGTYPE2ETYPE_MAP = { - 'string': 'String', - 'choice': 'String', - 'yn': 'Boolean', - 'int': 'Int', - 'float' : 'Float', - } - -_forced_mode = os.environ.get('CW_MODE') -assert _forced_mode in (None, 'system', 'user') - -# CWDEV tells whether directories such as i18n/, web/data/, etc. (ie containing -# some other resources than python libraries) are located with the python code -# or as a 'shared' cube -CWDEV = exists(join(CW_SOFTWARE_ROOT, 'i18n')) - -try: - _INSTALL_PREFIX = os.environ['CW_INSTALL_PREFIX'] -except KeyError: - _INSTALL_PREFIX = _find_prefix() -_USR_INSTALL = _INSTALL_PREFIX == '/usr' - -class CubicWebNoAppConfiguration(ConfigurationMixIn): - """base class for cubicweb configuration without a specific instance directory - """ - # to set in concrete configuration - name = None - # log messages format (see logging module documentation for available keys) - log_format = '%(asctime)s - (%(name)s) %(levelname)s: %(message)s' - # the format below can be useful to debug multi thread issues: - # log_format = '%(asctime)s - [%(threadName)s] (%(name)s) %(levelname)s: %(message)s' - # nor remove appobjects based on unused interface [???] - cleanup_unused_appobjects = True - - quick_start = False - - if (CWDEV and _forced_mode != 'system'): - mode = 'user' - _CUBES_DIR = join(CW_SOFTWARE_ROOT, '../cubes') - else: - mode = _forced_mode or 'system' - _CUBES_DIR = join(_INSTALL_PREFIX, 'share', 'cubicweb', 'cubes') - - CUBES_DIR = abspath(os.environ.get('CW_CUBES_DIR', _CUBES_DIR)) - CUBES_PATH = os.environ.get('CW_CUBES_PATH', '').split(os.pathsep) - - options = ( - ('log-threshold', - {'type' : 'string', # XXX use a dedicated type? - 'default': 'WARNING', - 'help': 'server\'s log level', - 'group': 'main', 'level': 1, - }), - ('umask', - {'type' : 'int', - 'default': 0o077, - 'help': 'permission umask for files created by the server', - 'group': 'main', 'level': 2, - }), - # common configuration options which are potentially required as soon as - # you're using "base" application objects (ie to really server/web - # specific) - ('base-url', - {'type' : 'string', - 'default': None, - 'help': 'web server root url', - 'group': 'main', 'level': 1, - }), - ('allow-email-login', - {'type' : 'yn', - 'default': False, - 'help': 'allow users to login with their primary email if set', - 'group': 'main', 'level': 2, - }), - ('mangle-emails', - {'type' : 'yn', - 'default': False, - 'help': "don't display actual email addresses but mangle them if \ -this option is set to yes", - 'group': 'email', 'level': 3, - }), - ) - # static and class methods used to get instance independant resources ## - @staticmethod - def cubicweb_version(): - """return installed cubicweb version""" - from logilab.common.changelog import Version - from cubicweb import __pkginfo__ - version = __pkginfo__.numversion - assert len(version) == 3, version - return Version(version) - - @staticmethod - def persistent_options_configuration(): - return Configuration(options=PERSISTENT_OPTIONS) - - @classmethod - def shared_dir(cls): - """return the shared data directory (i.e. directory where standard - library views and data may be found) - """ - if CWDEV: - return join(CW_SOFTWARE_ROOT, 'web') - return cls.cube_dir('shared') - - @classmethod - def i18n_lib_dir(cls): - """return instance's i18n directory""" - if CWDEV: - return join(CW_SOFTWARE_ROOT, 'i18n') - return join(cls.shared_dir(), 'i18n') - - @classmethod - def cw_languages(cls): - for fname in os.listdir(join(cls.i18n_lib_dir())): - if fname.endswith('.po'): - yield splitext(fname)[0] - - - @classmethod - def available_cubes(cls): - import re - cubes = set() - for directory in cls.cubes_search_path(): - if not exists(directory): - cls.error('unexistant directory in cubes search path: %s' - % directory) - continue - for cube in os.listdir(directory): - if cube == 'shared': - continue - if not re.match('[_A-Za-z][_A-Za-z0-9]*$', cube): - continue # skip invalid python package name - cubedir = join(directory, cube) - if isdir(cubedir) and exists(join(cubedir, '__init__.py')): - cubes.add(cube) - return sorted(cubes) - - @classmethod - def cubes_search_path(cls): - """return the path of directories where cubes should be searched""" - path = [abspath(normpath(directory)) for directory in cls.CUBES_PATH - if directory.strip() and exists(directory.strip())] - if not cls.CUBES_DIR in path and exists(cls.CUBES_DIR): - path.append(cls.CUBES_DIR) - return path - - @classproperty - def extrapath(cls): - extrapath = {} - for cubesdir in cls.cubes_search_path(): - if cubesdir != cls.CUBES_DIR: - extrapath[cubesdir] = 'cubes' - return extrapath - - @classmethod - def cube_dir(cls, cube): - """return the cube directory for the given cube id, raise - `ConfigurationError` if it doesn't exist - """ - for directory in cls.cubes_search_path(): - cubedir = join(directory, cube) - if exists(cubedir): - return cubedir - raise ConfigurationError('no cube %r in %s' % ( - cube, cls.cubes_search_path())) - - @classmethod - def cube_migration_scripts_dir(cls, cube): - """cube migration scripts directory""" - return join(cls.cube_dir(cube), 'migration') - - @classmethod - def cube_pkginfo(cls, cube): - """return the information module for the given cube""" - cube = CW_MIGRATION_MAP.get(cube, cube) - try: - parent = __import__('cubes.%s.__pkginfo__' % cube) - return getattr(parent, cube).__pkginfo__ - except Exception as ex: - raise ConfigurationError( - 'unable to find packaging information for cube %s (%s: %s)' - % (cube, ex.__class__.__name__, ex)) - - @classmethod - def cube_version(cls, cube): - """return the version of the cube located in the given directory - """ - from logilab.common.changelog import Version - version = cls.cube_pkginfo(cube).numversion - assert len(version) == 3, version - return Version(version) - - @classmethod - def _cube_deps(cls, cube, key, oldkey): - """return cubicweb cubes used by the given cube""" - pkginfo = cls.cube_pkginfo(cube) - try: - # explicit __xxx_cubes__ attribute - deps = getattr(pkginfo, key) - except AttributeError: - # deduce cubes from generic __xxx__ attribute - try: - gendeps = getattr(pkginfo, key.replace('_cubes', '')) - except AttributeError: - deps = {} - else: - deps = dict( (x[len('cubicweb-'):], v) - for x, v in gendeps.items() - if x.startswith('cubicweb-')) - for depcube in deps: - try: - newname = CW_MIGRATION_MAP[depcube] - except KeyError: - pass - else: - deps[newname] = deps.pop(depcube) - return deps - - @classmethod - def cube_depends_cubicweb_version(cls, cube): - # XXX no backward compat (see _cube_deps above) - try: - pkginfo = cls.cube_pkginfo(cube) - deps = getattr(pkginfo, '__depends__') - return deps.get('cubicweb') - except AttributeError: - return None - - @classmethod - def cube_dependencies(cls, cube): - """return cubicweb cubes used by the given cube""" - return cls._cube_deps(cube, '__depends_cubes__', '__use__') - - @classmethod - def cube_recommends(cls, cube): - """return cubicweb cubes recommended by the given cube""" - return cls._cube_deps(cube, '__recommends_cubes__', '__recommend__') - - @classmethod - def expand_cubes(cls, cubes, with_recommends=False): - """expand the given list of top level cubes used by adding recursivly - each cube dependencies - """ - cubes = list(cubes) - todo = cubes[:] - if with_recommends: - available = set(cls.available_cubes()) - while todo: - cube = todo.pop(0) - for depcube in cls.cube_dependencies(cube): - if depcube not in cubes: - cubes.append(depcube) - todo.append(depcube) - if with_recommends: - for depcube in cls.cube_recommends(cube): - if depcube not in cubes and depcube in available: - cubes.append(depcube) - todo.append(depcube) - return cubes - - @classmethod - def reorder_cubes(cls, cubes): - """reorder cubes from the top level cubes to inner dependencies - cubes - """ - from logilab.common.graph import ordered_nodes, UnorderableGraph - graph = {} - for cube in cubes: - cube = CW_MIGRATION_MAP.get(cube, cube) - graph[cube] = set(dep for dep in cls.cube_dependencies(cube) - if dep in cubes) - graph[cube] |= set(dep for dep in cls.cube_recommends(cube) - if dep in cubes) - try: - return ordered_nodes(graph) - except UnorderableGraph as ex: - raise ConfigurationError(ex) - - @classmethod - def cls_adjust_sys_path(cls): - """update python path if necessary""" - cubes_parent_dir = normpath(join(cls.CUBES_DIR, '..')) - if not cubes_parent_dir in sys.path: - sys.path.insert(0, cubes_parent_dir) - try: - import cubes - cubes.__path__ = cls.cubes_search_path() - except ImportError: - return # cubes dir doesn't exists - - @classmethod - def load_available_configs(cls): - from logilab.common.modutils import load_module_from_file - for conffile in ('web/webconfig.py', 'etwist/twconfig.py', - 'server/serverconfig.py',): - if exists(join(CW_SOFTWARE_ROOT, conffile)): - load_module_from_file(join(CW_SOFTWARE_ROOT, conffile)) - - @classmethod - def load_cwctl_plugins(cls): - from logilab.common.modutils import load_module_from_file - cls.cls_adjust_sys_path() - for ctlfile in ('web/webctl.py', 'etwist/twctl.py', - 'server/serverctl.py', - 'devtools/devctl.py', 'goa/goactl.py'): - if exists(join(CW_SOFTWARE_ROOT, ctlfile)): - try: - load_module_from_file(join(CW_SOFTWARE_ROOT, ctlfile)) - except ImportError as err: - cls.error('could not import the command provider %s: %s', - ctlfile, err) - cls.info('loaded cubicweb-ctl plugin %s', ctlfile) - for cube in cls.available_cubes(): - pluginfile = join(cls.cube_dir(cube), 'ccplugin.py') - initfile = join(cls.cube_dir(cube), '__init__.py') - if exists(pluginfile): - try: - __import__('cubes.%s.ccplugin' % cube) - cls.info('loaded cubicweb-ctl plugin from %s', cube) - except Exception: - cls.exception('while loading plugin %s', pluginfile) - elif exists(initfile): - try: - __import__('cubes.%s' % cube) - except Exception: - cls.exception('while loading cube %s', cube) - else: - cls.warning('no __init__ file in cube %s', cube) - - @classmethod - def init_available_cubes(cls): - """cubes may register some sources (svnfile for instance) in their - __init__ file, so they should be loaded early in the startup process - """ - for cube in cls.available_cubes(): - try: - __import__('cubes.%s' % cube) - except Exception as ex: - cls.warning("can't init cube %s: %s", cube, ex) - - cubicweb_appobject_path = set(['entities']) - cube_appobject_path = set(['entities']) - - def __init__(self, debugmode=False): - if debugmode: - # in python 2.7, DeprecationWarning are not shown anymore by default - filterwarnings('default', category=DeprecationWarning) - register_stored_procedures() - self._cubes = None - super(CubicWebNoAppConfiguration, self).__init__() - self.debugmode = debugmode - self.adjust_sys_path() - self.load_defaults() - # will be properly initialized later by _gettext_init - self.translations = {'en': (text_type, lambda ctx, msgid: text_type(msgid) )} - self._site_loaded = set() - # don't register ReStructured Text directives by simple import, avoid pb - # with eg sphinx. - # XXX should be done properly with a function from cw.uicfg - try: - from cubicweb.ext.rest import cw_rest_init - except ImportError: - pass - else: - cw_rest_init() - - def adjust_sys_path(self): - # overriden in CubicWebConfiguration - self.cls_adjust_sys_path() - - def init_log(self, logthreshold=None, logfile=None, syslog=False): - """init the log service""" - if logthreshold is None: - if self.debugmode: - logthreshold = 'DEBUG' - else: - logthreshold = self['log-threshold'] - if sys.platform == 'win32': - # no logrotate on win32, so use logging rotation facilities - # for now, hard code weekly rotation every sunday, and 52 weeks kept - # idea: make this configurable? - init_log(self.debugmode, syslog, logthreshold, logfile, self.log_format, - rotation_parameters={'when': 'W6', # every sunday - 'interval': 1, - 'backupCount': 52}) - else: - init_log(self.debugmode, syslog, logthreshold, logfile, self.log_format) - # configure simpleTal logger - logging.getLogger('simpleTAL').setLevel(logging.ERROR) - - def appobjects_path(self): - """return a list of files or directories where the registry will look - for application objects. By default return nothing in NoApp config. - """ - return [] - - def build_appobjects_path(self, templpath, evobjpath=None, tvobjpath=None): - """given a list of directories, return a list of sub files and - directories that should be loaded by the instance objects registry. - - :param evobjpath: - optional list of sub-directories (or files without the .py ext) of - the cubicweb library that should be tested and added to the output list - if they exists. If not give, default to `cubicweb_appobject_path` class - attribute. - :param tvobjpath: - optional list of sub-directories (or files without the .py ext) of - directories given in `templpath` that should be tested and added to - the output list if they exists. If not give, default to - `cube_appobject_path` class attribute. - """ - vregpath = self.build_appobjects_cubicweb_path(evobjpath) - vregpath += self.build_appobjects_cube_path(templpath, tvobjpath) - return vregpath - - def build_appobjects_cubicweb_path(self, evobjpath=None): - vregpath = [] - if evobjpath is None: - evobjpath = self.cubicweb_appobject_path - # NOTE: for the order, see http://www.cubicweb.org/ticket/2330799 - # it is clearly a workaround - for subdir in sorted(evobjpath, key=lambda x:x != 'entities'): - path = join(CW_SOFTWARE_ROOT, subdir) - if exists(path): - vregpath.append(path) - return vregpath - - def build_appobjects_cube_path(self, templpath, tvobjpath=None): - vregpath = [] - if tvobjpath is None: - tvobjpath = self.cube_appobject_path - for directory in templpath: - # NOTE: for the order, see http://www.cubicweb.org/ticket/2330799 - for subdir in sorted(tvobjpath, key=lambda x:x != 'entities'): - path = join(directory, subdir) - if exists(path): - vregpath.append(path) - elif exists(path + '.py'): - vregpath.append(path + '.py') - return vregpath - - apphome = None - - def load_site_cubicweb(self, paths=None): - """load instance's specific site_cubicweb file""" - if paths is None: - paths = self.cubes_path() - if self.apphome is not None: - paths = [self.apphome] + paths - for path in reversed(paths): - sitefile = join(path, 'site_cubicweb.py') - if exists(sitefile) and not sitefile in self._site_loaded: - self._load_site_cubicweb(sitefile) - self._site_loaded.add(sitefile) - - def _load_site_cubicweb(self, sitefile): - # XXX extrapath argument to load_module_from_file only in lgc > 0.50.2 - from logilab.common.modutils import load_module_from_modpath, modpath_from_file - module = load_module_from_modpath(modpath_from_file(sitefile, self.extrapath)) - self.debug('%s loaded', sitefile) - return module - - def cwproperty_definitions(self): - cfg = self.persistent_options_configuration() - for section, options in cfg.options_by_section(): - section = section.lower() - for optname, optdict, value in options: - key = '%s.%s' % (section, optname) - type, vocab = self.map_option(optdict) - default = cfg.option_default(optname, optdict) - pdef = {'type': type, 'vocabulary': vocab, 'default': default, - 'help': optdict['help'], - 'sitewide': optdict.get('sitewide', False)} - yield key, pdef - - def map_option(self, optdict): - try: - vocab = optdict['choices'] - except KeyError: - vocab = optdict.get('vocabulary') - if isinstance(vocab, Method): - vocab = getattr(self, vocab.method, ()) - return CFGTYPE2ETYPE_MAP[optdict['type']], vocab - - def default_instance_id(self): - """return the instance identifier, useful for option which need this - as default value - """ - return None - - _cubes = None - - def init_cubes(self, cubes): - self._cubes = self.reorder_cubes(cubes) - # load cubes'__init__.py file first - for cube in cubes: - __import__('cubes.%s' % cube) - self.load_site_cubicweb() - - def cubes(self): - """return the list of cubes used by this instance - - result is ordered from the top level cubes to inner dependencies - cubes - """ - assert self._cubes is not None, 'cubes not initialized' - return self._cubes - - def cubes_path(self): - """return the list of path to cubes used by this instance, from outer - most to inner most cubes - """ - return [self.cube_dir(p) for p in self.cubes()] - - # these are overridden by set_log_methods below - # only defining here to prevent pylint from complaining - @classmethod - def debug(cls, msg, *a, **kw): - pass - info = warning = error = critical = exception = debug - - -class CubicWebConfiguration(CubicWebNoAppConfiguration): - """base class for cubicweb server and web configurations""" - - if CubicWebNoAppConfiguration.mode == 'user': - _INSTANCES_DIR = expanduser('~/etc/cubicweb.d/') - #mode == system' - elif _USR_INSTALL: - _INSTANCES_DIR = '/etc/cubicweb.d/' - else: - _INSTANCES_DIR = join(_INSTALL_PREFIX, 'etc', 'cubicweb.d') - - # set to true during repair (shell, migration) to allow some things which - # wouldn't be possible otherwise - repairing = False - - # set by upgrade command - verbosity = 0 - cmdline_options = None - options = CubicWebNoAppConfiguration.options + ( - ('log-file', - {'type' : 'string', - 'default': Method('default_log_file'), - 'help': 'file where output logs should be written', - 'group': 'main', 'level': 2, - }), - ('statsd-endpoint', - {'type' : 'string', - 'default': '', - 'help': 'UDP address of the statsd endpoint; it must be formatted' - 'like :; disabled is unset.', - 'group': 'main', 'level': 2, - }), - # email configuration - ('smtp-host', - {'type' : 'string', - 'default': 'mail', - 'help': 'hostname of the SMTP mail server', - 'group': 'email', 'level': 1, - }), - ('smtp-port', - {'type' : 'int', - 'default': 25, - 'help': 'listening port of the SMTP mail server', - 'group': 'email', 'level': 1, - }), - ('sender-name', - {'type' : 'string', - 'default': Method('default_instance_id'), - 'help': 'name used as HELO name for outgoing emails from the \ -repository.', - 'group': 'email', 'level': 2, - }), - ('sender-addr', - {'type' : 'string', - 'default': 'cubicweb@mydomain.com', - 'help': 'email address used as HELO address for outgoing emails from \ -the repository', - 'group': 'email', 'level': 1, - }), - ('logstat-interval', - {'type' : 'int', - 'default': 0, - 'help': 'interval (in seconds) at which stats are dumped in the logstat file; set 0 to disable', - 'group': 'main', 'level': 2, - }), - ('logstat-file', - {'type' : 'string', - 'default': Method('default_stats_file'), - 'help': 'file where stats for the instance should be written', - 'group': 'main', 'level': 2, - }), - ) - - @classmethod - def instances_dir(cls): - """return the control directory""" - return abspath(os.environ.get('CW_INSTANCES_DIR', cls._INSTANCES_DIR)) - - @classmethod - def migration_scripts_dir(cls): - """cubicweb migration scripts directory""" - if CWDEV: - return join(CW_SOFTWARE_ROOT, 'misc', 'migration') - mdir = join(_INSTALL_PREFIX, 'share', 'cubicweb', 'migration') - if not exists(mdir): - raise ConfigurationError('migration path %s doesn\'t exist' % mdir) - return mdir - - @classmethod - def config_for(cls, appid, config=None, debugmode=False, creating=False): - """return a configuration instance for the given instance identifier - """ - cls.load_available_configs() - config = config or guess_configuration(cls.instance_home(appid)) - configcls = configuration_cls(config) - return configcls(appid, debugmode, creating) - - @classmethod - def possible_configurations(cls, appid): - """return the name of possible configurations for the given - instance id - """ - home = cls.instance_home(appid) - return possible_configurations(home) - - @classmethod - def instance_home(cls, appid): - """return the home directory of the instance with the given - instance id - """ - home = join(cls.instances_dir(), appid) - if not exists(home): - raise ConfigurationError('no such instance %s (check it exists with' - ' "cubicweb-ctl list")' % appid) - return home - - MODES = ('common', 'repository', 'Any') - MCOMPAT = {'all-in-one': MODES, - 'repository': ('common', 'repository', 'Any')} - @classmethod - def accept_mode(cls, mode): - #assert mode in cls.MODES, mode - return mode in cls.MCOMPAT[cls.name] - - # default configuration methods ########################################### - - def default_instance_id(self): - """return the instance identifier, useful for option which need this - as default value - """ - return self.appid - - def default_log_file(self): - """return default path to the log file of the instance'server""" - if self.mode == 'user': - import tempfile - basepath = join(tempfile.gettempdir(), '%s-%s' % ( - basename(self.appid), self.name)) - path = basepath + '.log' - i = 1 - while exists(path) and i < 100: # arbitrary limit to avoid infinite loop - try: - open(path, 'a') - break - except IOError: - path = '%s-%s.log' % (basepath, i) - i += 1 - return path - if _USR_INSTALL: - return '/var/log/cubicweb/%s-%s.log' % (self.appid, self.name) - else: - log_path = os.path.join(_INSTALL_PREFIX, 'var', 'log', 'cubicweb', '%s-%s.log') - return log_path % (self.appid, self.name) - - def default_stats_file(self): - """return default path to the stats file of the instance'server""" - logfile = self.default_log_file() - if logfile.endswith('.log'): - logfile = logfile[:-4] - return logfile + '.stats' - - def default_pid_file(self): - """return default path to the pid file of the instance'server""" - if self.mode == 'system': - if _USR_INSTALL: - default = '/var/run/cubicweb/' - else: - default = os.path.join(_INSTALL_PREFIX, 'var', 'run', 'cubicweb') - else: - import tempfile - default = tempfile.gettempdir() - # runtime directory created on startup if necessary, don't check it - # exists - rtdir = abspath(os.environ.get('CW_RUNTIME_DIR', default)) - return join(rtdir, '%s-%s.pid' % (self.appid, self.name)) - - # config -> repository - - def repository(self, vreg=None): - from cubicweb.server.repository import Repository - from cubicweb.server.utils import TasksManager - return Repository(self, TasksManager(), vreg=vreg) - - # instance methods used to get instance specific resources ############# - - def __init__(self, appid, debugmode=False, creating=False): - self.appid = appid - # set to true while creating an instance - self.creating = creating - super(CubicWebConfiguration, self).__init__(debugmode) - fake_gettext = (text_type, lambda ctx, msgid: text_type(msgid)) - for lang in self.available_languages(): - self.translations[lang] = fake_gettext - self._cubes = None - self.load_file_configuration(self.main_config_file()) - - def adjust_sys_path(self): - super(CubicWebConfiguration, self).adjust_sys_path() - # adding apphome to python path is not usually necessary in production - # environments, but necessary for tests - if self.apphome and self.apphome not in sys.path: - sys.path.insert(0, self.apphome) - - @property - def apphome(self): - return join(self.instances_dir(), self.appid) - - @property - def appdatahome(self): - if self.mode == 'system': - if _USR_INSTALL: - iddir = os.path.join('/var','lib', 'cubicweb', 'instances') - else: - iddir = os.path.join(_INSTALL_PREFIX, 'var', 'lib', 'cubicweb', 'instances') - else: - iddir = self.instances_dir() - iddir = abspath(os.environ.get('CW_INSTANCES_DATA_DIR', iddir)) - return join(iddir, self.appid) - - def init_cubes(self, cubes): - super(CubicWebConfiguration, self).init_cubes(cubes) - # reload config file in cases options are defined in cubes __init__ - # or site_cubicweb files - self.load_file_configuration(self.main_config_file()) - # configuration initialization hook - self.load_configuration(**(self.cmdline_options or {})) - - def add_cubes(self, cubes): - """add given cubes to the list of used cubes""" - if not isinstance(cubes, list): - cubes = list(cubes) - self._cubes = self.reorder_cubes(list(self._cubes) + cubes) - self.load_site_cubicweb([self.cube_dir(cube) for cube in cubes]) - - def main_config_file(self): - """return instance's control configuration file""" - return join(self.apphome, '%s.conf' % self.name) - - def save(self): - """write down current configuration""" - with open(self.main_config_file(), 'w') as fobj: - self.generate_config(fobj) - - def check_writeable_uid_directory(self, path): - """check given directory path exists, belongs to the user running the - server process and is writeable. - - If not, try to fix this, letting exception propagate when not possible. - """ - if not exists(path): - self.info('creating %s directory', path) - try: - os.makedirs(path) - except OSError as ex: - self.warning('error while creating %s directory: %s', path, ex) - return - if self['uid']: - try: - uid = int(self['uid']) - except ValueError: - from pwd import getpwnam - uid = getpwnam(self['uid']).pw_uid - else: - try: - uid = os.getuid() - except AttributeError: # we are on windows - return - fstat = os.stat(path) - if fstat.st_uid != uid: - self.info('giving ownership of %s directory to %s', path, self['uid']) - try: - os.chown(path, uid, os.getgid()) - except OSError as ex: - self.warning('error while giving ownership of %s directory to %s: %s', - path, self['uid'], ex) - if not (fstat.st_mode & stat.S_IWUSR): - self.info('forcing write permission on directory %s', path) - try: - os.chmod(path, fstat.st_mode | stat.S_IWUSR) - except OSError as ex: - self.warning('error while forcing write permission on directory %s: %s', - path, ex) - return - - @cached - def instance_md5_version(self): - from hashlib import md5 # pylint: disable=E0611 - infos = [] - for pkg in sorted(self.cubes()): - version = self.cube_version(pkg) - infos.append('%s-%s' % (pkg, version)) - infos.append('cubicweb-%s' % str(self.cubicweb_version())) - return md5((';'.join(infos)).encode('ascii')).hexdigest() - - def load_configuration(self, **kw): - """load instance's configuration files""" - super(CubicWebConfiguration, self).load_configuration(**kw) - if self.apphome and not self.creating: - # init gettext - self._gettext_init() - - def _load_site_cubicweb(self, sitefile): - # overridden to register cube specific options - mod = super(CubicWebConfiguration, self)._load_site_cubicweb(sitefile) - if getattr(mod, 'options', None): - self.register_options(mod.options) - self.load_defaults() - - def init_log(self, logthreshold=None, force=False): - """init the log service""" - if not force and hasattr(self, '_logging_initialized'): - return - self._logging_initialized = True - super_self = super(CubicWebConfiguration, self) - super_self.init_log(logthreshold, logfile=self.get('log-file')) - # read a config file if it exists - logconfig = join(self.apphome, 'logging.conf') - if exists(logconfig): - logging.config.fileConfig(logconfig) - # set the statsd address, if any - if self.get('statsd-endpoint'): - try: - address, port = self.get('statsd-endpoint').split(':') - port = int(port) - except: - self.error('statsd-endpoint: invalid address format ({}); ' - 'it should be "ip:port"'.format(self.get('statsd-endpoint'))) - else: - import statsd_logger - statsd_logger.setup('cubicweb.%s' % self.appid, (address, port)) - - def available_languages(self, *args): - """return available translation for an instance, by looking for - compiled catalog - - take \*args to be usable as a vocabulary method - """ - from glob import glob - yield 'en' # ensure 'en' is yielded even if no .mo found - for path in glob(join(self.apphome, 'i18n', - '*', 'LC_MESSAGES')): - lang = path.split(os.sep)[-2] - if lang != 'en': - yield lang - - def _gettext_init(self): - """set language for gettext""" - from cubicweb.cwgettext import translation - path = join(self.apphome, 'i18n') - for language in self.available_languages(): - self.info("loading language %s", language) - try: - tr = translation('cubicweb', path, languages=[language]) - self.translations[language] = (tr.ugettext, tr.upgettext) - except (ImportError, AttributeError, IOError): - if self.mode != 'test': - # in test contexts, data/i18n does not exist, hence - # logging will only pollute the logs - self.exception('localisation support error for language %s', - language) - - def appobjects_path(self): - """return a list of files or directories where the registry will look - for application objects - """ - templpath = list(reversed(self.cubes_path())) - if self.apphome: # may be unset in tests - templpath.append(self.apphome) - return self.build_appobjects_path(templpath) - - def set_sources_mode(self, sources): - if not 'all' in sources: - print('warning: ignoring specified sources, requires a repository ' - 'configuration') - - def i18ncompile(self, langs=None): - from cubicweb import i18n - if langs is None: - langs = self.available_languages() - i18ndir = join(self.apphome, 'i18n') - if not exists(i18ndir): - create_dir(i18ndir) - sourcedirs = [join(path, 'i18n') for path in self.cubes_path()] - sourcedirs.append(self.i18n_lib_dir()) - return i18n.compile_i18n_catalogs(sourcedirs, i18ndir, langs) - - def sendmails(self, msgs, fromaddr=None): - """msgs: list of 2-uple (message object, recipients). Return False - if connection to the smtp server failed, else True. - """ - server, port = self['smtp-host'], self['smtp-port'] - if fromaddr is None: - fromaddr = '%s <%s>' % (self['sender-name'], self['sender-addr']) - SMTP_LOCK.acquire() - try: - try: - smtp = SMTP(server, port) - except Exception as ex: - self.exception("can't connect to smtp server %s:%s (%s)", - server, port, ex) - return False - for msg, recipients in msgs: - try: - smtp.sendmail(fromaddr, recipients, msg.as_string()) - except Exception as ex: - self.exception("error sending mail to %s (%s)", - recipients, ex) - smtp.close() - finally: - SMTP_LOCK.release() - return True - -set_log_methods(CubicWebNoAppConfiguration, - logging.getLogger('cubicweb.configuration')) - -# alias to get a configuration instance from an instance id -instance_configuration = CubicWebConfiguration.config_for -application_configuration = deprecated('use instance_configuration')(instance_configuration) - - -_EXT_REGISTERED = False -def register_stored_procedures(): - from logilab.database import FunctionDescr - from rql.utils import register_function, iter_funcnode_variables - from rql.nodes import SortTerm, Constant, VariableRef - - global _EXT_REGISTERED - if _EXT_REGISTERED: - return - _EXT_REGISTERED = True - - class COMMA_JOIN(FunctionDescr): - supported_backends = ('postgres', 'sqlite',) - rtype = 'String' - - def st_description(self, funcnode, mainindex, tr): - return ', '.join(sorted(term.get_description(mainindex, tr) - for term in iter_funcnode_variables(funcnode))) - - register_function(COMMA_JOIN) # XXX do not expose? - - - class CONCAT_STRINGS(COMMA_JOIN): - aggregat = True - - register_function(CONCAT_STRINGS) # XXX bw compat - - - class GROUP_CONCAT(CONCAT_STRINGS): - supported_backends = ('mysql', 'postgres', 'sqlite',) - - register_function(GROUP_CONCAT) - - - class LIMIT_SIZE(FunctionDescr): - supported_backends = ('postgres', 'sqlite',) - minargs = maxargs = 3 - rtype = 'String' - - def st_description(self, funcnode, mainindex, tr): - return funcnode.children[0].get_description(mainindex, tr) - - register_function(LIMIT_SIZE) - - - class TEXT_LIMIT_SIZE(LIMIT_SIZE): - supported_backends = ('mysql', 'postgres', 'sqlite',) - minargs = maxargs = 2 - - register_function(TEXT_LIMIT_SIZE) - - - class FTIRANK(FunctionDescr): - """return ranking of a variable that must be used as some has_text - relation subject in the query's restriction. Usually used to sort result - of full-text search by ranking. - """ - supported_backends = ('postgres',) - rtype = 'Float' - - def st_check_backend(self, backend, funcnode): - """overriden so that on backend not supporting fti ranking, the - function is removed when in an orderby clause, or replaced by a 1.0 - constant. - """ - if not self.supports(backend): - parent = funcnode.parent - while parent is not None and not isinstance(parent, SortTerm): - parent = parent.parent - if isinstance(parent, SortTerm): - parent.parent.remove(parent) - else: - funcnode.parent.replace(funcnode, Constant(1.0, 'Float')) - parent = funcnode - for vref in parent.iget_nodes(VariableRef): - vref.unregister_reference() - - register_function(FTIRANK) - - - class FSPATH(FunctionDescr): - """return path of some bytes attribute stored using the Bytes - File-System Storage (bfss) - """ - rtype = 'Bytes' # XXX return a String? potential pb with fs encoding - - def update_cb_stack(self, stack): - assert len(stack) == 1 - stack[0] = self.source_execute - - def as_sql(self, backend, args): - raise NotImplementedError( - 'This callback is only available for BytesFileSystemStorage ' - 'managed attribute. Is FSPATH() argument BFSS managed?') - - def source_execute(self, source, session, value): - fpath = source.binary_to_str(value) - try: - return Binary(fpath) - except OSError as ex: - source.critical("can't open %s: %s", fpath, ex) - return None - - register_function(FSPATH)