--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/cubicweb/migration.py Sat Jan 16 13:48:51 2016 +0100
@@ -0,0 +1,553 @@
+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of CubicWeb.
+#
+# CubicWeb is free software: you can redistribute it and/or modify it under the
+# terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option)
+# any later version.
+#
+# CubicWeb is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with CubicWeb. If not, see <http://www.gnu.org/licenses/>.
+"""utilities for instances migration"""
+from __future__ import print_function
+
+__docformat__ = "restructuredtext en"
+
+import sys
+import os
+import logging
+import tempfile
+from os.path import exists, join, basename, splitext
+from itertools import chain
+from warnings import warn
+
+from six import string_types
+
+from logilab.common import IGNORED_EXTENSIONS
+from logilab.common.decorators import cached
+from logilab.common.configuration import REQUIRED, read_old_config
+from logilab.common.shellutils import ASK
+from logilab.common.changelog import Version
+from logilab.common.deprecation import deprecated
+
+from cubicweb import ConfigurationError, ExecutionError
+from cubicweb.cwconfig import CubicWebConfiguration as cwcfg
+from cubicweb.toolsutils import show_diffs
+
+def filter_scripts(config, directory, fromversion, toversion, quiet=True):
+ """return a list of paths of migration files to consider to upgrade
+ from a version to a greater one
+ """
+ from logilab.common.changelog import Version # doesn't work with appengine
+ assert fromversion
+ assert toversion
+ assert isinstance(fromversion, tuple), fromversion.__class__
+ assert isinstance(toversion, tuple), toversion.__class__
+ assert fromversion <= toversion, (fromversion, toversion)
+ if not exists(directory):
+ if not quiet:
+ print(directory, "doesn't exists, no migration path")
+ return []
+ if fromversion == toversion:
+ return []
+ result = []
+ for fname in os.listdir(directory):
+ if fname.endswith(IGNORED_EXTENSIONS):
+ continue
+ fpath = join(directory, fname)
+ try:
+ tver, mode = fname.split('_', 1)
+ except ValueError:
+ continue
+ mode = mode.split('.', 1)[0]
+ if not config.accept_mode(mode):
+ continue
+ try:
+ tver = Version(tver)
+ except ValueError:
+ continue
+ if tver <= fromversion:
+ continue
+ if tver > toversion:
+ continue
+ result.append((tver, fpath))
+ # be sure scripts are executed in order
+ return sorted(result)
+
+
+def execscript_confirm(scriptpath):
+ """asks for confirmation before executing a script and provides the
+ ability to show the script's content
+ """
+ while True:
+ answer = ASK.ask('Execute %r ?' % scriptpath,
+ ('Y','n','show','abort'), 'Y')
+ if answer == 'abort':
+ raise SystemExit(1)
+ elif answer == 'n':
+ return False
+ elif answer == 'show':
+ stream = open(scriptpath)
+ scriptcontent = stream.read()
+ stream.close()
+ print()
+ print(scriptcontent)
+ print()
+ else:
+ return True
+
+def yes(*args, **kwargs):
+ return True
+
+
+class MigrationHelper(object):
+ """class holding CubicWeb Migration Actions used by migration scripts"""
+
+ def __init__(self, config, interactive=True, verbosity=1):
+ self.config = config
+ if config:
+ # no config on shell to a remote instance
+ self.config.init_log(logthreshold=logging.ERROR)
+ # 0: no confirmation, 1: only main commands confirmed, 2 ask for everything
+ self.verbosity = verbosity
+ self.need_wrap = True
+ if not interactive or not verbosity:
+ self.confirm = yes
+ self.execscript_confirm = yes
+ else:
+ self.execscript_confirm = execscript_confirm
+ self._option_changes = []
+ self.__context = {'confirm': self.confirm,
+ 'config': self.config,
+ 'interactive_mode': interactive,
+ }
+ self._context_stack = []
+
+ def __getattribute__(self, name):
+ try:
+ return object.__getattribute__(self, name)
+ except AttributeError:
+ cmd = 'cmd_%s' % name
+ # search self.__class__ to avoid infinite recursion
+ if hasattr(self.__class__, cmd):
+ meth = getattr(self, cmd)
+ return lambda *args, **kwargs: self.interact(args, kwargs,
+ meth=meth)
+ raise
+ raise AttributeError(name)
+
+ def migrate(self, vcconf, toupgrade, options):
+ """upgrade the given set of cubes
+
+ `cubes` is an ordered list of 3-uple:
+ (cube, fromversion, toversion)
+ """
+ if options.fs_only:
+ # monkey path configuration.accept_mode so database mode (e.g. Any)
+ # won't be accepted
+ orig_accept_mode = self.config.accept_mode
+ def accept_mode(mode):
+ if mode == 'Any':
+ return False
+ return orig_accept_mode(mode)
+ self.config.accept_mode = accept_mode
+ # may be an iterator
+ toupgrade = tuple(toupgrade)
+ vmap = dict( (cube, (fromver, tover)) for cube, fromver, tover in toupgrade)
+ ctx = self.__context
+ ctx['versions_map'] = vmap
+ if self.config.accept_mode('Any') and 'cubicweb' in vmap:
+ migrdir = self.config.migration_scripts_dir()
+ self.cmd_process_script(join(migrdir, 'bootstrapmigration_repository.py'))
+ for cube, fromversion, toversion in toupgrade:
+ if cube == 'cubicweb':
+ migrdir = self.config.migration_scripts_dir()
+ else:
+ migrdir = self.config.cube_migration_scripts_dir(cube)
+ scripts = filter_scripts(self.config, migrdir, fromversion, toversion)
+ if scripts:
+ prevversion = None
+ for version, script in scripts:
+ # take care to X.Y.Z_Any.py / X.Y.Z_common.py: we've to call
+ # cube_upgraded once all script of X.Y.Z have been executed
+ if prevversion is not None and version != prevversion:
+ self.cube_upgraded(cube, prevversion)
+ prevversion = version
+ self.cmd_process_script(script)
+ self.cube_upgraded(cube, toversion)
+ else:
+ self.cube_upgraded(cube, toversion)
+
+ def cube_upgraded(self, cube, version):
+ pass
+
+ def shutdown(self):
+ pass
+
+ def interact(self, args, kwargs, meth):
+ """execute the given method according to user's confirmation"""
+ msg = 'Execute command: %s(%s) ?' % (
+ meth.__name__[4:],
+ ', '.join([repr(arg) for arg in args] +
+ ['%s=%r' % (n,v) for n,v in kwargs.items()]))
+ if 'ask_confirm' in kwargs:
+ ask_confirm = kwargs.pop('ask_confirm')
+ else:
+ ask_confirm = True
+ if not ask_confirm or self.confirm(msg):
+ return meth(*args, **kwargs)
+
+ def confirm(self, question, # pylint: disable=E0202
+ shell=True, abort=True, retry=False, pdb=False, default='y'):
+ """ask for confirmation and return true on positive answer
+
+ if `retry` is true the r[etry] answer may return 2
+ """
+ possibleanswers = ['y', 'n']
+ if abort:
+ possibleanswers.append('abort')
+ if pdb:
+ possibleanswers.append('pdb')
+ if shell:
+ possibleanswers.append('shell')
+ if retry:
+ possibleanswers.append('retry')
+ try:
+ answer = ASK.ask(question, possibleanswers, default)
+ except (EOFError, KeyboardInterrupt):
+ answer = 'abort'
+ if answer == 'n':
+ return False
+ if answer == 'retry':
+ return 2
+ if answer == 'abort':
+ raise SystemExit(1)
+ if answer == 'shell':
+ self.interactive_shell()
+ return self.confirm(question, shell, abort, retry, pdb, default)
+ if answer == 'pdb':
+ import pdb
+ pdb.set_trace()
+ return self.confirm(question, shell, abort, retry, pdb, default)
+ return True
+
+ def interactive_shell(self):
+ self.confirm = yes
+ self.need_wrap = False
+ # avoid '_' to be added to builtins by sys.display_hook
+ def do_not_add___to_builtins(obj):
+ if obj is not None:
+ print(repr(obj))
+ sys.displayhook = do_not_add___to_builtins
+ local_ctx = self._create_context()
+ try:
+ import readline
+ from cubicweb.toolsutils import CWShellCompleter
+ except ImportError:
+ # readline not available
+ pass
+ else:
+ rql_completer = CWShellCompleter(local_ctx)
+ readline.set_completer(rql_completer.complete)
+ readline.parse_and_bind('tab: complete')
+ home_key = 'HOME'
+ if sys.platform == 'win32':
+ home_key = 'USERPROFILE'
+ histfile = os.path.join(os.environ[home_key], ".cwshell_history")
+ try:
+ readline.read_history_file(histfile)
+ except IOError:
+ pass
+ from code import interact
+ banner = """entering the migration python shell
+just type migration commands or arbitrary python code and type ENTER to execute it
+type "exit" or Ctrl-D to quit the shell and resume operation"""
+ # give custom readfunc to avoid http://bugs.python.org/issue1288615
+ def unicode_raw_input(prompt):
+ return unicode(raw_input(prompt), sys.stdin.encoding)
+ interact(banner, readfunc=unicode_raw_input, local=local_ctx)
+ try:
+ readline.write_history_file(histfile)
+ except IOError:
+ pass
+ # delete instance's confirm attribute to avoid questions
+ del self.confirm
+ self.need_wrap = True
+
+ @cached
+ def _create_context(self):
+ """return a dictionary to use as migration script execution context"""
+ context = self.__context
+ for attr in dir(self):
+ if attr.startswith('cmd_'):
+ if self.need_wrap:
+ context[attr[4:]] = getattr(self, attr[4:])
+ else:
+ context[attr[4:]] = getattr(self, attr)
+ return context
+
+ def update_context(self, key, value):
+ for context in self._context_stack:
+ context[key] = value
+ self.__context[key] = value
+
+ def cmd_process_script(self, migrscript, funcname=None, *args, **kwargs):
+ """execute a migration script in interactive mode
+
+ Display the migration script path, ask for confirmation and execute it
+ if confirmed
+
+ Allowed input file formats for migration scripts:
+ - `python` (.py)
+ - `sql` (.sql)
+ - `doctest` (.txt or .rst)
+
+ .. warning:: sql migration scripts are not available in web-only instance
+
+ You can pass script parameters with using double dash (--) in the
+ command line
+
+ Context environment can have these variables defined:
+ - __name__ : will be determine by funcname parameter
+ - __file__ : is the name of the script if it exists
+ - __args__ : script arguments coming from command-line
+
+ :param migrscript: name of the script
+ :param funcname: defines __name__ inside the shell (or use __main__)
+ :params args: optional arguments for funcname
+ :keyword scriptargs: optional arguments of the script
+ """
+ ftypes = {'python': ('.py',),
+ 'doctest': ('.txt', '.rst'),
+ 'sql': ('.sql',)}
+ # sql migration scripts are not available in web-only instance
+ if not hasattr(self, "session"):
+ ftypes.pop('sql')
+ migrscript = os.path.normpath(migrscript)
+ for (script_mode, ftype) in ftypes.items():
+ if migrscript.endswith(ftype):
+ break
+ else:
+ ftypes = ', '.join(chain(*ftypes.values()))
+ msg = 'ignoring %s, not a valid script extension (%s)'
+ raise ExecutionError(msg % (migrscript, ftypes))
+ if not self.execscript_confirm(migrscript):
+ return
+ scriptlocals = self._create_context().copy()
+ scriptlocals.update({'__file__': migrscript,
+ '__args__': kwargs.pop("scriptargs", [])})
+ self._context_stack.append(scriptlocals)
+ if script_mode == 'python':
+ if funcname is None:
+ pyname = '__main__'
+ else:
+ pyname = splitext(basename(migrscript))[0]
+ scriptlocals['__name__'] = pyname
+ with open(migrscript, 'rb') as fobj:
+ fcontent = fobj.read()
+ try:
+ code = compile(fcontent, migrscript, 'exec')
+ except SyntaxError:
+ # try without print_function
+ code = compile(fcontent, migrscript, 'exec', 0, True)
+ warn('[3.22] script %r should be updated to work with print_function'
+ % migrscript, DeprecationWarning)
+ exec(code, scriptlocals)
+ if funcname is not None:
+ try:
+ func = scriptlocals[funcname]
+ self.info('found %s in locals', funcname)
+ assert callable(func), '%s (%s) is not callable' % (func, funcname)
+ except KeyError:
+ self.critical('no %s in script %s', funcname, migrscript)
+ return None
+ return func(*args, **kwargs)
+ elif script_mode == 'sql':
+ from cubicweb.server.sqlutils import sqlexec
+ sqlexec(open(migrscript).read(), self.session.system_sql)
+ self.commit()
+ else: # script_mode == 'doctest'
+ import doctest
+ return doctest.testfile(migrscript, module_relative=False,
+ optionflags=doctest.ELLIPSIS,
+ # verbose mode when user input is expected
+ verbose=self.verbosity==2,
+ report=True,
+ encoding='utf-8',
+ globs=scriptlocals)
+ self._context_stack.pop()
+
+ def cmd_option_renamed(self, oldname, newname):
+ """a configuration option has been renamed"""
+ self._option_changes.append(('renamed', oldname, newname))
+
+ def cmd_option_group_changed(self, option, oldgroup, newgroup):
+ """a configuration option has been moved in another group"""
+ self._option_changes.append(('moved', option, oldgroup, newgroup))
+
+ def cmd_option_added(self, optname):
+ """a configuration option has been added"""
+ self._option_changes.append(('added', optname))
+
+ def cmd_option_removed(self, optname):
+ """a configuration option has been removed"""
+ # can safely be ignored
+ #self._option_changes.append(('removed', optname))
+
+ def cmd_option_type_changed(self, optname, oldtype, newvalue):
+ """a configuration option's type has changed"""
+ self._option_changes.append(('typechanged', optname, oldtype, newvalue))
+
+ def cmd_add_cubes(self, cubes):
+ """modify the list of used cubes in the in-memory config
+ returns newly inserted cubes, including dependencies
+ """
+ if isinstance(cubes, string_types):
+ cubes = (cubes,)
+ origcubes = self.config.cubes()
+ newcubes = [p for p in self.config.expand_cubes(cubes)
+ if not p in origcubes]
+ if newcubes:
+ self.config.add_cubes(newcubes)
+ return newcubes
+
+ @deprecated('[3.20] use drop_cube() instead of remove_cube()')
+ def cmd_remove_cube(self, cube, removedeps=False):
+ return self.cmd_drop_cube(cube, removedeps)
+
+ def cmd_drop_cube(self, cube, removedeps=False):
+ if removedeps:
+ toremove = self.config.expand_cubes([cube])
+ else:
+ toremove = (cube,)
+ origcubes = self.config._cubes
+ basecubes = [c for c in origcubes if not c in toremove]
+ # don't fake-add any new ones, or we won't be able to really-add them later
+ self.config._cubes = tuple(cube for cube in self.config.expand_cubes(basecubes)
+ if cube in origcubes)
+ removed = [p for p in origcubes if not p in self.config._cubes]
+ if not cube in removed and cube in origcubes:
+ raise ConfigurationError("can't remove cube %s, "
+ "used as a dependency" % cube)
+ return removed
+
+ def rewrite_configuration(self):
+ configfile = self.config.main_config_file()
+ if self._option_changes:
+ read_old_config(self.config, self._option_changes, configfile)
+ fd, newconfig = tempfile.mkstemp()
+ for optdescr in self._option_changes:
+ if optdescr[0] == 'added':
+ optdict = self.config.get_option_def(optdescr[1])
+ if optdict.get('default') is REQUIRED:
+ self.config.input_option(optdescr[1], optdict)
+ self.config.generate_config(open(newconfig, 'w'))
+ show_diffs(configfile, newconfig, askconfirm=self.confirm is not yes)
+ os.close(fd)
+ if exists(newconfig):
+ os.unlink(newconfig)
+
+ # these are overridden by set_log_methods below
+ # only defining here to prevent pylint from complaining
+ info = warning = error = critical = exception = debug = lambda msg,*a,**kw: None
+
+from logging import getLogger
+from cubicweb import set_log_methods
+set_log_methods(MigrationHelper, getLogger('cubicweb.migration'))
+
+
+def version_strictly_lower(a, b):
+ if a is None:
+ return True
+ if b is None:
+ return False
+ if a:
+ a = Version(a)
+ if b:
+ b = Version(b)
+ return a < b
+
+def max_version(a, b):
+ return str(max(Version(a), Version(b)))
+
+class ConfigurationProblem(object):
+ """Each cube has its own list of dependencies on other cubes/versions.
+
+ The ConfigurationProblem is used to record the loaded cubes, then to detect
+ inconsistencies in their dependencies.
+
+ See configuration management on wikipedia for litterature.
+ """
+
+ def __init__(self, config):
+ self.config = config
+ self.cubes = {'cubicweb': cwcfg.cubicweb_version()}
+
+ def add_cube(self, name, version):
+ self.cubes[name] = version
+
+ def solve(self):
+ self.warnings = []
+ self.errors = []
+ self.dependencies = {}
+ self.reverse_dependencies = {}
+ self.constraints = {}
+ # read dependencies
+ for cube in self.cubes:
+ if cube == 'cubicweb': continue
+ self.dependencies[cube] = dict(self.config.cube_dependencies(cube))
+ self.dependencies[cube]['cubicweb'] = self.config.cube_depends_cubicweb_version(cube)
+ # compute reverse dependencies
+ for cube, dependencies in self.dependencies.items():
+ for name, constraint in dependencies.items():
+ self.reverse_dependencies.setdefault(name,set())
+ if constraint:
+ try:
+ oper, version = constraint.split()
+ self.reverse_dependencies[name].add( (oper, version, cube) )
+ except Exception:
+ self.warnings.append(
+ 'cube %s depends on %s but constraint badly '
+ 'formatted: %s' % (cube, name, constraint))
+ else:
+ self.reverse_dependencies[name].add( (None, None, cube) )
+ # check consistency
+ for cube, versions in sorted(self.reverse_dependencies.items()):
+ oper, version, source = None, None, None
+ # simplify constraints
+ if versions:
+ for constraint in versions:
+ op, ver, src = constraint
+ if oper is None:
+ oper = op
+ version = ver
+ source = src
+ elif op == '>=' and oper == '>=':
+ if version_strictly_lower(version, ver):
+ version = ver
+ source = src
+ elif op == None:
+ continue
+ else:
+ print('unable to handle %s in %s, set to `%s %s` '
+ 'but currently up to `%s %s`' %
+ (cube, source, oper, version, op, ver))
+ # "solve" constraint satisfaction problem
+ if cube not in self.cubes:
+ self.errors.append( ('add', cube, version, source) )
+ elif versions:
+ lower_strict = version_strictly_lower(self.cubes[cube], version)
+ if oper in ('>=','=','=='):
+ if lower_strict:
+ self.errors.append( ('update', cube, version, source) )
+ elif oper is None:
+ pass # no constraint on version
+ else:
+ print('unknown operator', oper)