diff -r 058bb3dc685f -r 0b59724cb3f2 migration.py --- a/migration.py Mon Jan 04 18:40:30 2016 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,553 +0,0 @@ -# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved. -# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr -# -# This file is part of CubicWeb. -# -# CubicWeb is free software: you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation, either version 2.1 of the License, or (at your option) -# any later version. -# -# CubicWeb is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License along -# with CubicWeb. If not, see . -"""utilities for instances migration""" -from __future__ import print_function - -__docformat__ = "restructuredtext en" - -import sys -import os -import logging -import tempfile -from os.path import exists, join, basename, splitext -from itertools import chain -from warnings import warn - -from six import string_types - -from logilab.common import IGNORED_EXTENSIONS -from logilab.common.decorators import cached -from logilab.common.configuration import REQUIRED, read_old_config -from logilab.common.shellutils import ASK -from logilab.common.changelog import Version -from logilab.common.deprecation import deprecated - -from cubicweb import ConfigurationError, ExecutionError -from cubicweb.cwconfig import CubicWebConfiguration as cwcfg -from cubicweb.toolsutils import show_diffs - -def filter_scripts(config, directory, fromversion, toversion, quiet=True): - """return a list of paths of migration files to consider to upgrade - from a version to a greater one - """ - from logilab.common.changelog import Version # doesn't work with appengine - assert fromversion - assert toversion - assert isinstance(fromversion, tuple), fromversion.__class__ - assert isinstance(toversion, tuple), toversion.__class__ - assert fromversion <= toversion, (fromversion, toversion) - if not exists(directory): - if not quiet: - print(directory, "doesn't exists, no migration path") - return [] - if fromversion == toversion: - return [] - result = [] - for fname in os.listdir(directory): - if fname.endswith(IGNORED_EXTENSIONS): - continue - fpath = join(directory, fname) - try: - tver, mode = fname.split('_', 1) - except ValueError: - continue - mode = mode.split('.', 1)[0] - if not config.accept_mode(mode): - continue - try: - tver = Version(tver) - except ValueError: - continue - if tver <= fromversion: - continue - if tver > toversion: - continue - result.append((tver, fpath)) - # be sure scripts are executed in order - return sorted(result) - - -def execscript_confirm(scriptpath): - """asks for confirmation before executing a script and provides the - ability to show the script's content - """ - while True: - answer = ASK.ask('Execute %r ?' % scriptpath, - ('Y','n','show','abort'), 'Y') - if answer == 'abort': - raise SystemExit(1) - elif answer == 'n': - return False - elif answer == 'show': - stream = open(scriptpath) - scriptcontent = stream.read() - stream.close() - print() - print(scriptcontent) - print() - else: - return True - -def yes(*args, **kwargs): - return True - - -class MigrationHelper(object): - """class holding CubicWeb Migration Actions used by migration scripts""" - - def __init__(self, config, interactive=True, verbosity=1): - self.config = config - if config: - # no config on shell to a remote instance - self.config.init_log(logthreshold=logging.ERROR) - # 0: no confirmation, 1: only main commands confirmed, 2 ask for everything - self.verbosity = verbosity - self.need_wrap = True - if not interactive or not verbosity: - self.confirm = yes - self.execscript_confirm = yes - else: - self.execscript_confirm = execscript_confirm - self._option_changes = [] - self.__context = {'confirm': self.confirm, - 'config': self.config, - 'interactive_mode': interactive, - } - self._context_stack = [] - - def __getattribute__(self, name): - try: - return object.__getattribute__(self, name) - except AttributeError: - cmd = 'cmd_%s' % name - # search self.__class__ to avoid infinite recursion - if hasattr(self.__class__, cmd): - meth = getattr(self, cmd) - return lambda *args, **kwargs: self.interact(args, kwargs, - meth=meth) - raise - raise AttributeError(name) - - def migrate(self, vcconf, toupgrade, options): - """upgrade the given set of cubes - - `cubes` is an ordered list of 3-uple: - (cube, fromversion, toversion) - """ - if options.fs_only: - # monkey path configuration.accept_mode so database mode (e.g. Any) - # won't be accepted - orig_accept_mode = self.config.accept_mode - def accept_mode(mode): - if mode == 'Any': - return False - return orig_accept_mode(mode) - self.config.accept_mode = accept_mode - # may be an iterator - toupgrade = tuple(toupgrade) - vmap = dict( (cube, (fromver, tover)) for cube, fromver, tover in toupgrade) - ctx = self.__context - ctx['versions_map'] = vmap - if self.config.accept_mode('Any') and 'cubicweb' in vmap: - migrdir = self.config.migration_scripts_dir() - self.cmd_process_script(join(migrdir, 'bootstrapmigration_repository.py')) - for cube, fromversion, toversion in toupgrade: - if cube == 'cubicweb': - migrdir = self.config.migration_scripts_dir() - else: - migrdir = self.config.cube_migration_scripts_dir(cube) - scripts = filter_scripts(self.config, migrdir, fromversion, toversion) - if scripts: - prevversion = None - for version, script in scripts: - # take care to X.Y.Z_Any.py / X.Y.Z_common.py: we've to call - # cube_upgraded once all script of X.Y.Z have been executed - if prevversion is not None and version != prevversion: - self.cube_upgraded(cube, prevversion) - prevversion = version - self.cmd_process_script(script) - self.cube_upgraded(cube, toversion) - else: - self.cube_upgraded(cube, toversion) - - def cube_upgraded(self, cube, version): - pass - - def shutdown(self): - pass - - def interact(self, args, kwargs, meth): - """execute the given method according to user's confirmation""" - msg = 'Execute command: %s(%s) ?' % ( - meth.__name__[4:], - ', '.join([repr(arg) for arg in args] + - ['%s=%r' % (n,v) for n,v in kwargs.items()])) - if 'ask_confirm' in kwargs: - ask_confirm = kwargs.pop('ask_confirm') - else: - ask_confirm = True - if not ask_confirm or self.confirm(msg): - return meth(*args, **kwargs) - - def confirm(self, question, # pylint: disable=E0202 - shell=True, abort=True, retry=False, pdb=False, default='y'): - """ask for confirmation and return true on positive answer - - if `retry` is true the r[etry] answer may return 2 - """ - possibleanswers = ['y', 'n'] - if abort: - possibleanswers.append('abort') - if pdb: - possibleanswers.append('pdb') - if shell: - possibleanswers.append('shell') - if retry: - possibleanswers.append('retry') - try: - answer = ASK.ask(question, possibleanswers, default) - except (EOFError, KeyboardInterrupt): - answer = 'abort' - if answer == 'n': - return False - if answer == 'retry': - return 2 - if answer == 'abort': - raise SystemExit(1) - if answer == 'shell': - self.interactive_shell() - return self.confirm(question, shell, abort, retry, pdb, default) - if answer == 'pdb': - import pdb - pdb.set_trace() - return self.confirm(question, shell, abort, retry, pdb, default) - return True - - def interactive_shell(self): - self.confirm = yes - self.need_wrap = False - # avoid '_' to be added to builtins by sys.display_hook - def do_not_add___to_builtins(obj): - if obj is not None: - print(repr(obj)) - sys.displayhook = do_not_add___to_builtins - local_ctx = self._create_context() - try: - import readline - from cubicweb.toolsutils import CWShellCompleter - except ImportError: - # readline not available - pass - else: - rql_completer = CWShellCompleter(local_ctx) - readline.set_completer(rql_completer.complete) - readline.parse_and_bind('tab: complete') - home_key = 'HOME' - if sys.platform == 'win32': - home_key = 'USERPROFILE' - histfile = os.path.join(os.environ[home_key], ".cwshell_history") - try: - readline.read_history_file(histfile) - except IOError: - pass - from code import interact - banner = """entering the migration python shell -just type migration commands or arbitrary python code and type ENTER to execute it -type "exit" or Ctrl-D to quit the shell and resume operation""" - # give custom readfunc to avoid http://bugs.python.org/issue1288615 - def unicode_raw_input(prompt): - return unicode(raw_input(prompt), sys.stdin.encoding) - interact(banner, readfunc=unicode_raw_input, local=local_ctx) - try: - readline.write_history_file(histfile) - except IOError: - pass - # delete instance's confirm attribute to avoid questions - del self.confirm - self.need_wrap = True - - @cached - def _create_context(self): - """return a dictionary to use as migration script execution context""" - context = self.__context - for attr in dir(self): - if attr.startswith('cmd_'): - if self.need_wrap: - context[attr[4:]] = getattr(self, attr[4:]) - else: - context[attr[4:]] = getattr(self, attr) - return context - - def update_context(self, key, value): - for context in self._context_stack: - context[key] = value - self.__context[key] = value - - def cmd_process_script(self, migrscript, funcname=None, *args, **kwargs): - """execute a migration script in interactive mode - - Display the migration script path, ask for confirmation and execute it - if confirmed - - Allowed input file formats for migration scripts: - - `python` (.py) - - `sql` (.sql) - - `doctest` (.txt or .rst) - - .. warning:: sql migration scripts are not available in web-only instance - - You can pass script parameters with using double dash (--) in the - command line - - Context environment can have these variables defined: - - __name__ : will be determine by funcname parameter - - __file__ : is the name of the script if it exists - - __args__ : script arguments coming from command-line - - :param migrscript: name of the script - :param funcname: defines __name__ inside the shell (or use __main__) - :params args: optional arguments for funcname - :keyword scriptargs: optional arguments of the script - """ - ftypes = {'python': ('.py',), - 'doctest': ('.txt', '.rst'), - 'sql': ('.sql',)} - # sql migration scripts are not available in web-only instance - if not hasattr(self, "session"): - ftypes.pop('sql') - migrscript = os.path.normpath(migrscript) - for (script_mode, ftype) in ftypes.items(): - if migrscript.endswith(ftype): - break - else: - ftypes = ', '.join(chain(*ftypes.values())) - msg = 'ignoring %s, not a valid script extension (%s)' - raise ExecutionError(msg % (migrscript, ftypes)) - if not self.execscript_confirm(migrscript): - return - scriptlocals = self._create_context().copy() - scriptlocals.update({'__file__': migrscript, - '__args__': kwargs.pop("scriptargs", [])}) - self._context_stack.append(scriptlocals) - if script_mode == 'python': - if funcname is None: - pyname = '__main__' - else: - pyname = splitext(basename(migrscript))[0] - scriptlocals['__name__'] = pyname - with open(migrscript, 'rb') as fobj: - fcontent = fobj.read() - try: - code = compile(fcontent, migrscript, 'exec') - except SyntaxError: - # try without print_function - code = compile(fcontent, migrscript, 'exec', 0, True) - warn('[3.22] script %r should be updated to work with print_function' - % migrscript, DeprecationWarning) - exec(code, scriptlocals) - if funcname is not None: - try: - func = scriptlocals[funcname] - self.info('found %s in locals', funcname) - assert callable(func), '%s (%s) is not callable' % (func, funcname) - except KeyError: - self.critical('no %s in script %s', funcname, migrscript) - return None - return func(*args, **kwargs) - elif script_mode == 'sql': - from cubicweb.server.sqlutils import sqlexec - sqlexec(open(migrscript).read(), self.session.system_sql) - self.commit() - else: # script_mode == 'doctest' - import doctest - return doctest.testfile(migrscript, module_relative=False, - optionflags=doctest.ELLIPSIS, - # verbose mode when user input is expected - verbose=self.verbosity==2, - report=True, - encoding='utf-8', - globs=scriptlocals) - self._context_stack.pop() - - def cmd_option_renamed(self, oldname, newname): - """a configuration option has been renamed""" - self._option_changes.append(('renamed', oldname, newname)) - - def cmd_option_group_changed(self, option, oldgroup, newgroup): - """a configuration option has been moved in another group""" - self._option_changes.append(('moved', option, oldgroup, newgroup)) - - def cmd_option_added(self, optname): - """a configuration option has been added""" - self._option_changes.append(('added', optname)) - - def cmd_option_removed(self, optname): - """a configuration option has been removed""" - # can safely be ignored - #self._option_changes.append(('removed', optname)) - - def cmd_option_type_changed(self, optname, oldtype, newvalue): - """a configuration option's type has changed""" - self._option_changes.append(('typechanged', optname, oldtype, newvalue)) - - def cmd_add_cubes(self, cubes): - """modify the list of used cubes in the in-memory config - returns newly inserted cubes, including dependencies - """ - if isinstance(cubes, string_types): - cubes = (cubes,) - origcubes = self.config.cubes() - newcubes = [p for p in self.config.expand_cubes(cubes) - if not p in origcubes] - if newcubes: - self.config.add_cubes(newcubes) - return newcubes - - @deprecated('[3.20] use drop_cube() instead of remove_cube()') - def cmd_remove_cube(self, cube, removedeps=False): - return self.cmd_drop_cube(cube, removedeps) - - def cmd_drop_cube(self, cube, removedeps=False): - if removedeps: - toremove = self.config.expand_cubes([cube]) - else: - toremove = (cube,) - origcubes = self.config._cubes - basecubes = [c for c in origcubes if not c in toremove] - # don't fake-add any new ones, or we won't be able to really-add them later - self.config._cubes = tuple(cube for cube in self.config.expand_cubes(basecubes) - if cube in origcubes) - removed = [p for p in origcubes if not p in self.config._cubes] - if not cube in removed and cube in origcubes: - raise ConfigurationError("can't remove cube %s, " - "used as a dependency" % cube) - return removed - - def rewrite_configuration(self): - configfile = self.config.main_config_file() - if self._option_changes: - read_old_config(self.config, self._option_changes, configfile) - fd, newconfig = tempfile.mkstemp() - for optdescr in self._option_changes: - if optdescr[0] == 'added': - optdict = self.config.get_option_def(optdescr[1]) - if optdict.get('default') is REQUIRED: - self.config.input_option(optdescr[1], optdict) - self.config.generate_config(open(newconfig, 'w')) - show_diffs(configfile, newconfig, askconfirm=self.confirm is not yes) - os.close(fd) - if exists(newconfig): - os.unlink(newconfig) - - # these are overridden by set_log_methods below - # only defining here to prevent pylint from complaining - info = warning = error = critical = exception = debug = lambda msg,*a,**kw: None - -from logging import getLogger -from cubicweb import set_log_methods -set_log_methods(MigrationHelper, getLogger('cubicweb.migration')) - - -def version_strictly_lower(a, b): - if a is None: - return True - if b is None: - return False - if a: - a = Version(a) - if b: - b = Version(b) - return a < b - -def max_version(a, b): - return str(max(Version(a), Version(b))) - -class ConfigurationProblem(object): - """Each cube has its own list of dependencies on other cubes/versions. - - The ConfigurationProblem is used to record the loaded cubes, then to detect - inconsistencies in their dependencies. - - See configuration management on wikipedia for litterature. - """ - - def __init__(self, config): - self.config = config - self.cubes = {'cubicweb': cwcfg.cubicweb_version()} - - def add_cube(self, name, version): - self.cubes[name] = version - - def solve(self): - self.warnings = [] - self.errors = [] - self.dependencies = {} - self.reverse_dependencies = {} - self.constraints = {} - # read dependencies - for cube in self.cubes: - if cube == 'cubicweb': continue - self.dependencies[cube] = dict(self.config.cube_dependencies(cube)) - self.dependencies[cube]['cubicweb'] = self.config.cube_depends_cubicweb_version(cube) - # compute reverse dependencies - for cube, dependencies in self.dependencies.items(): - for name, constraint in dependencies.items(): - self.reverse_dependencies.setdefault(name,set()) - if constraint: - try: - oper, version = constraint.split() - self.reverse_dependencies[name].add( (oper, version, cube) ) - except Exception: - self.warnings.append( - 'cube %s depends on %s but constraint badly ' - 'formatted: %s' % (cube, name, constraint)) - else: - self.reverse_dependencies[name].add( (None, None, cube) ) - # check consistency - for cube, versions in sorted(self.reverse_dependencies.items()): - oper, version, source = None, None, None - # simplify constraints - if versions: - for constraint in versions: - op, ver, src = constraint - if oper is None: - oper = op - version = ver - source = src - elif op == '>=' and oper == '>=': - if version_strictly_lower(version, ver): - version = ver - source = src - elif op == None: - continue - else: - print('unable to handle %s in %s, set to `%s %s` ' - 'but currently up to `%s %s`' % - (cube, source, oper, version, op, ver)) - # "solve" constraint satisfaction problem - if cube not in self.cubes: - self.errors.append( ('add', cube, version, source) ) - elif versions: - lower_strict = version_strictly_lower(self.cubes[cube], version) - if oper in ('>=','=','=='): - if lower_strict: - self.errors.append( ('update', cube, version, source) ) - elif oper is None: - pass # no constraint on version - else: - print('unknown operator', oper)