1 # copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.

     2 # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr

     3 #

     4 # This file is part of CubicWeb.

     5 #

     6 # CubicWeb is free software: you can redistribute it and/or modify it under the

     7 # terms of the GNU Lesser General Public License as published by the Free

     8 # Software Foundation, either version 2.1 of the License, or (at your option)

     9 # any later version.

    10 #

    11 # CubicWeb is distributed in the hope that it will be useful, but WITHOUT

    12 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS

    13 # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more

    14 # details.

    15 #

    16 # You should have received a copy of the GNU Lesser General Public License along

    17 # with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.

    18 """utilities for instances migration"""

    19 from __future__ import print_function

    20 

    21 __docformat__ = "restructuredtext en"

    22 

    23 import sys

    24 import os

    25 import logging

    26 import tempfile

    27 from os.path import exists, join, basename, splitext

    28 from itertools import chain

    29 from warnings import warn

    30 

    31 from six import string_types

    32 

    33 from logilab.common import IGNORED_EXTENSIONS

    34 from logilab.common.decorators import cached

    35 from logilab.common.configuration import REQUIRED, read_old_config

    36 from logilab.common.shellutils import ASK

    37 from logilab.common.changelog import Version

    38 from logilab.common.deprecation import deprecated

    39 

    40 from cubicweb import ConfigurationError, ExecutionError

    41 from cubicweb.cwconfig import CubicWebConfiguration as cwcfg

    42 from cubicweb.toolsutils import show_diffs

    43 

    44 def filter_scripts(config, directory, fromversion, toversion, quiet=True):

    45     """return a list of paths of migration files to consider to upgrade

    46     from a version to a greater one

    47     """

    48     from logilab.common.changelog import Version # doesn't work with appengine

    49     assert fromversion

    50     assert toversion

    51     assert isinstance(fromversion, tuple), fromversion.__class__

    52     assert isinstance(toversion, tuple), toversion.__class__

    53     assert fromversion <= toversion, (fromversion, toversion)

    54     if not exists(directory):

    55         if not quiet:

    56             print(directory, "doesn't exists, no migration path")

    57         return []

    58     if fromversion == toversion:

    59         return []

    60     result = []

    61     for fname in os.listdir(directory):

    62         if fname.endswith(IGNORED_EXTENSIONS):

    63             continue

    64         fpath = join(directory, fname)

    65         try:

    66             tver, mode = fname.split('_', 1)

    67         except ValueError:

    68             continue

    69         mode = mode.split('.', 1)[0]

    70         if not config.accept_mode(mode):

    71             continue

    72         try:

    73             tver = Version(tver)

    74         except ValueError:

    75             continue

    76         if tver <= fromversion:

    77             continue

    78         if tver > toversion:

    79             continue

    80         result.append((tver, fpath))

    81     # be sure scripts are executed in order

    82     return sorted(result)

    83 

    84 

    85 def execscript_confirm(scriptpath):

    86     """asks for confirmation before executing a script and provides the

    87     ability to show the script's content

    88     """

    89     while True:

    90         answer = ASK.ask('Execute %r ?' % scriptpath,

    91                          ('Y','n','show','abort'), 'Y')

    92         if answer == 'abort':

    93             raise SystemExit(1)

    94         elif answer == 'n':

    95             return False

    96         elif answer == 'show':

    97             stream = open(scriptpath)

    98             scriptcontent = stream.read()

    99             stream.close()

   100             print()

   101             print(scriptcontent)

   102             print()

   103         else:

   104             return True

   105 

   106 def yes(*args, **kwargs):

   107     return True

   108 

   109 

   110 class MigrationHelper(object):

   111     """class holding CubicWeb Migration Actions used by migration scripts"""

   112 

   113     def __init__(self, config, interactive=True, verbosity=1):

   114         self.config = config

   115         if config:

   116             # no config on shell to a remote instance

   117             self.config.init_log(logthreshold=logging.ERROR)

   118         # 0: no confirmation, 1: only main commands confirmed, 2 ask for everything

   119         self.verbosity = verbosity

   120         self.need_wrap = True

   121         if not interactive or not verbosity:

   122             self.confirm = yes

   123             self.execscript_confirm = yes

   124         else:

   125             self.execscript_confirm = execscript_confirm

   126         self._option_changes = []

   127         self.__context = {'confirm': self.confirm,

   128                           'config': self.config,

   129                           'interactive_mode': interactive,

   130                           }

   131         self._context_stack = []

   132 

   133     def __getattribute__(self, name):

   134         try:

   135             return object.__getattribute__(self, name)

   136         except AttributeError:

   137             cmd = 'cmd_%s' % name

   138             # search self.__class__ to avoid infinite recursion

   139             if hasattr(self.__class__, cmd):

   140                 meth = getattr(self, cmd)

   141                 return lambda *args, **kwargs: self.interact(args, kwargs,

   142                                                              meth=meth)

   143             raise

   144         raise AttributeError(name)

   145 

   146     def migrate(self, vcconf, toupgrade, options):

   147         """upgrade the given set of cubes

   148 

   149         `cubes` is an ordered list of 3-uple:

   150         (cube, fromversion, toversion)

   151         """

   152         if options.fs_only:

   153             # monkey path configuration.accept_mode so database mode (e.g. Any)

   154             # won't be accepted

   155             orig_accept_mode = self.config.accept_mode

   156             def accept_mode(mode):

   157                 if mode == 'Any':

   158                     return False

   159                 return orig_accept_mode(mode)

   160             self.config.accept_mode = accept_mode

   161         # may be an iterator

   162         toupgrade = tuple(toupgrade)

   163         vmap = dict( (cube, (fromver, tover)) for cube, fromver, tover in toupgrade)

   164         ctx = self.__context

   165         ctx['versions_map'] = vmap

   166         if self.config.accept_mode('Any') and 'cubicweb' in vmap:

   167             migrdir = self.config.migration_scripts_dir()

   168             self.cmd_process_script(join(migrdir, 'bootstrapmigration_repository.py'))

   169         for cube, fromversion, toversion in toupgrade:

   170             if cube == 'cubicweb':

   171                 migrdir = self.config.migration_scripts_dir()

   172             else:

   173                 migrdir = self.config.cube_migration_scripts_dir(cube)

   174             scripts = filter_scripts(self.config, migrdir, fromversion, toversion)

   175             if scripts:

   176                 prevversion = None

   177                 for version, script in scripts:

   178                     # take care to X.Y.Z_Any.py / X.Y.Z_common.py: we've to call

   179                     # cube_upgraded once all script of X.Y.Z have been executed

   180                     if prevversion is not None and version != prevversion:

   181                         self.cube_upgraded(cube, prevversion)

   182                     prevversion = version

   183                     self.cmd_process_script(script)

   184                 self.cube_upgraded(cube, toversion)

   185             else:

   186                 self.cube_upgraded(cube, toversion)

   187 

   188     def cube_upgraded(self, cube, version):

   189         pass

   190 

   191     def shutdown(self):

   192         pass

   193 

   194     def interact(self, args, kwargs, meth):

   195         """execute the given method according to user's confirmation"""

   196         msg = 'Execute command: %s(%s) ?' % (

   197             meth.__name__[4:],

   198             ', '.join([repr(arg) for arg in args] +

   199                       ['%s=%r' % (n,v) for n,v in kwargs.items()]))

   200         if 'ask_confirm' in kwargs:

   201             ask_confirm = kwargs.pop('ask_confirm')

   202         else:

   203             ask_confirm = True

   204         if not ask_confirm or self.confirm(msg):

   205             return meth(*args, **kwargs)

   206 

   207     def confirm(self, question, # pylint: disable=E0202

   208                 shell=True, abort=True, retry=False, pdb=False, default='y'):

   209         """ask for confirmation and return true on positive answer

   210 

   211         if `retry` is true the r[etry] answer may return 2

   212         """

   213         possibleanswers = ['y', 'n']

   214         if abort:

   215             possibleanswers.append('abort')

   216         if pdb:

   217             possibleanswers.append('pdb')

   218         if shell:

   219             possibleanswers.append('shell')

   220         if retry:

   221             possibleanswers.append('retry')

   222         try:

   223             answer = ASK.ask(question, possibleanswers, default)

   224         except (EOFError, KeyboardInterrupt):

   225             answer = 'abort'

   226         if answer == 'n':

   227             return False

   228         if answer == 'retry':

   229             return 2

   230         if answer == 'abort':

   231             raise SystemExit(1)

   232         if answer == 'shell':

   233             self.interactive_shell()

   234             return self.confirm(question, shell, abort, retry, pdb, default)

   235         if answer == 'pdb':

   236             import pdb

   237             pdb.set_trace()

   238             return self.confirm(question, shell, abort, retry, pdb, default)

   239         return True

   240 

   241     def interactive_shell(self):

   242         self.confirm = yes

   243         self.need_wrap = False

   244         # avoid '_' to be added to builtins by sys.display_hook

   245         def do_not_add___to_builtins(obj):

   246             if obj is not None:

   247                 print(repr(obj))

   248         sys.displayhook = do_not_add___to_builtins

   249         local_ctx = self._create_context()

   250         try:

   251             import readline

   252             from cubicweb.toolsutils import CWShellCompleter

   253         except ImportError:

   254             # readline not available

   255             pass

   256         else:

   257             rql_completer = CWShellCompleter(local_ctx)

   258             readline.set_completer(rql_completer.complete)

   259             readline.parse_and_bind('tab: complete')

   260             home_key = 'HOME'

   261             if sys.platform == 'win32':

   262                 home_key = 'USERPROFILE'

   263             histfile = os.path.join(os.environ[home_key], ".cwshell_history")

   264             try:

   265                 readline.read_history_file(histfile)

   266             except IOError:

   267                 pass

   268         from code import interact

   269         banner = """entering the migration python shell

   270 just type migration commands or arbitrary python code and type ENTER to execute it

   271 type "exit" or Ctrl-D to quit the shell and resume operation"""

   272         # give custom readfunc to avoid http://bugs.python.org/issue1288615

   273         def unicode_raw_input(prompt):

   274             return unicode(raw_input(prompt), sys.stdin.encoding)

   275         interact(banner, readfunc=unicode_raw_input, local=local_ctx)

   276         try:

   277             readline.write_history_file(histfile)

   278         except IOError:

   279             pass

   280         # delete instance's confirm attribute to avoid questions

   281         del self.confirm

   282         self.need_wrap = True

   283 

   284     @cached

   285     def _create_context(self):

   286         """return a dictionary to use as migration script execution context"""

   287         context = self.__context

   288         for attr in dir(self):

   289             if attr.startswith('cmd_'):

   290                 if self.need_wrap:

   291                     context[attr[4:]] = getattr(self, attr[4:])

   292                 else:

   293                     context[attr[4:]] = getattr(self, attr)

   294         return context

   295 

   296     def update_context(self, key, value):

   297         for context in self._context_stack:

   298             context[key] = value

   299         self.__context[key] = value

   300 

   301     def cmd_process_script(self, migrscript, funcname=None, *args, **kwargs):

   302         """execute a migration script in interactive mode

   303 

   304         Display the migration script path, ask for confirmation and execute it

   305         if confirmed

   306 

   307         Allowed input file formats for migration scripts:

   308         - `python` (.py)

   309         - `sql` (.sql)

   310         - `doctest` (.txt or .rst)

   311 

   312         .. warning:: sql migration scripts are not available in web-only instance

   313 

   314         You can pass script parameters with using double dash (--) in the

   315         command line

   316 

   317         Context environment can have these variables defined:

   318         - __name__ : will be determine by funcname parameter

   319         - __file__ : is the name of the script if it exists

   320         - __args__ : script arguments coming from command-line

   321 

   322         :param migrscript: name of the script

   323         :param funcname: defines __name__ inside the shell (or use __main__)

   324         :params args: optional arguments for funcname

   325         :keyword scriptargs: optional arguments of the script

   326         """

   327         ftypes = {'python':  ('.py',),

   328                   'doctest': ('.txt', '.rst'),

   329                   'sql':     ('.sql',)}

   330         # sql migration scripts are not available in web-only instance

   331         if not hasattr(self, "session"):

   332             ftypes.pop('sql')

   333         migrscript = os.path.normpath(migrscript)

   334         for (script_mode, ftype) in ftypes.items():

   335             if migrscript.endswith(ftype):

   336                 break

   337         else:

   338             ftypes = ', '.join(chain(*ftypes.values()))

   339             msg = 'ignoring %s, not a valid script extension (%s)'

   340             raise ExecutionError(msg % (migrscript, ftypes))

   341         if not self.execscript_confirm(migrscript):

   342             return

   343         scriptlocals = self._create_context().copy()

   344         scriptlocals.update({'__file__': migrscript,

   345                              '__args__': kwargs.pop("scriptargs", [])})

   346         self._context_stack.append(scriptlocals)

   347         if script_mode == 'python':

   348             if funcname is None:

   349                 pyname = '__main__'

   350             else:

   351                 pyname = splitext(basename(migrscript))[0]

   352             scriptlocals['__name__'] = pyname

   353             with open(migrscript, 'rb') as fobj:

   354                 fcontent = fobj.read()

   355             try:

   356                 code = compile(fcontent, migrscript, 'exec')

   357             except SyntaxError:

   358                 # try without print_function

   359                 code = compile(fcontent, migrscript, 'exec', 0, True)

   360                 warn('[3.22] script %r should be updated to work with print_function'

   361                      % migrscript, DeprecationWarning)

   362             exec(code, scriptlocals)

   363             if funcname is not None:

   364                 try:

   365                     func = scriptlocals[funcname]

   366                     self.info('found %s in locals', funcname)

   367                     assert callable(func), '%s (%s) is not callable' % (func, funcname)

   368                 except KeyError:

   369                     self.critical('no %s in script %s', funcname, migrscript)

   370                     return None

   371                 return func(*args, **kwargs)

   372         elif script_mode == 'sql':

   373             from cubicweb.server.sqlutils import sqlexec

   374             sqlexec(open(migrscript).read(), self.session.system_sql)

   375             self.commit()

   376         else: # script_mode == 'doctest'

   377             import doctest

   378             return doctest.testfile(migrscript, module_relative=False,

   379                                     optionflags=doctest.ELLIPSIS,

   380                                     # verbose mode when user input is expected

   381                                     verbose=self.verbosity==2,

   382                                     report=True,

   383                                     encoding='utf-8',

   384                                     globs=scriptlocals)

   385         self._context_stack.pop()

   386 

   387     def cmd_option_renamed(self, oldname, newname):

   388         """a configuration option has been renamed"""

   389         self._option_changes.append(('renamed', oldname, newname))

   390 

   391     def cmd_option_group_changed(self, option, oldgroup, newgroup):

   392         """a configuration option has been moved in another group"""

   393         self._option_changes.append(('moved', option, oldgroup, newgroup))

   394 

   395     def cmd_option_added(self, optname):

   396         """a configuration option has been added"""

   397         self._option_changes.append(('added', optname))

   398 

   399     def cmd_option_removed(self, optname):

   400         """a configuration option has been removed"""

   401         # can safely be ignored

   402         #self._option_changes.append(('removed', optname))

   403 

   404     def cmd_option_type_changed(self, optname, oldtype, newvalue):

   405         """a configuration option's type has changed"""

   406         self._option_changes.append(('typechanged', optname, oldtype, newvalue))

   407 

   408     def cmd_add_cubes(self, cubes):

   409         """modify the list of used cubes in the in-memory config

   410         returns newly inserted cubes, including dependencies

   411         """

   412         if isinstance(cubes, string_types):

   413             cubes = (cubes,)

   414         origcubes = self.config.cubes()

   415         newcubes = [p for p in self.config.expand_cubes(cubes)

   416                     if not p in origcubes]

   417         if newcubes:

   418             self.config.add_cubes(newcubes)

   419         return newcubes

   420 

   421     @deprecated('[3.20] use drop_cube() instead of remove_cube()')

   422     def cmd_remove_cube(self, cube, removedeps=False):

   423         return self.cmd_drop_cube(cube, removedeps)

   424 

   425     def cmd_drop_cube(self, cube, removedeps=False):

   426         if removedeps:

   427             toremove = self.config.expand_cubes([cube])

   428         else:

   429             toremove = (cube,)

   430         origcubes = self.config._cubes

   431         basecubes = [c for c in origcubes if not c in toremove]

   432         # don't fake-add any new ones, or we won't be able to really-add them later

   433         self.config._cubes = tuple(cube for cube in self.config.expand_cubes(basecubes)

   434                                    if cube in origcubes)

   435         removed = [p for p in origcubes if not p in self.config._cubes]

   436         if not cube in removed and cube in origcubes:

   437             raise ConfigurationError("can't remove cube %s, "

   438                                      "used as a dependency" % cube)

   439         return removed

   440 

   441     def rewrite_configuration(self):

   442         configfile = self.config.main_config_file()

   443         if self._option_changes:

   444             read_old_config(self.config, self._option_changes, configfile)

   445         fd, newconfig = tempfile.mkstemp()

   446         for optdescr in self._option_changes:

   447             if optdescr[0] == 'added':

   448                 optdict = self.config.get_option_def(optdescr[1])

   449                 if optdict.get('default') is REQUIRED:

   450                     self.config.input_option(optdescr[1], optdict)

   451         self.config.generate_config(open(newconfig, 'w'))

   452         show_diffs(configfile, newconfig, askconfirm=self.confirm is not yes)

   453         os.close(fd)

   454         if exists(newconfig):

   455             os.unlink(newconfig)

   456 

   457     # these are overridden by set_log_methods below

   458     # only defining here to prevent pylint from complaining

   459     info = warning = error = critical = exception = debug = lambda msg,*a,**kw: None

   460 

   461 from logging import getLogger

   462 from cubicweb import set_log_methods

   463 set_log_methods(MigrationHelper, getLogger('cubicweb.migration'))

   464 

   465 

   466 def version_strictly_lower(a, b):

   467     if a is None:

   468         return True

   469     if b is None:

   470         return False

   471     if a:

   472         a = Version(a)

   473     if b:

   474         b = Version(b)

   475     return a < b

   476 

   477 def max_version(a, b):

   478     return str(max(Version(a), Version(b)))

   479 

   480 class ConfigurationProblem(object):

   481     """Each cube has its own list of dependencies on other cubes/versions.

   482 

   483     The ConfigurationProblem is used to record the loaded cubes, then to detect

   484     inconsistencies in their dependencies.

   485 

   486     See configuration management on wikipedia for litterature.

   487     """

   488 

   489     def __init__(self, config):

   490         self.config = config

   491         self.cubes = {'cubicweb': cwcfg.cubicweb_version()}

   492 

   493     def add_cube(self, name, version):

   494         self.cubes[name] = version

   495 

   496     def solve(self):

   497         self.warnings = []

   498         self.errors = []

   499         self.dependencies = {}

   500         self.reverse_dependencies = {}

   501         self.constraints = {}

   502         # read dependencies

   503         for cube in self.cubes:

   504             if cube == 'cubicweb': continue

   505             self.dependencies[cube] = dict(self.config.cube_dependencies(cube))

   506             self.dependencies[cube]['cubicweb'] = self.config.cube_depends_cubicweb_version(cube)

   507         # compute reverse dependencies

   508         for cube, dependencies in self.dependencies.items():

   509             for name, constraint in dependencies.items():

   510                 self.reverse_dependencies.setdefault(name,set())

   511                 if constraint:

   512                     try:

   513                         oper, version = constraint.split()

   514                         self.reverse_dependencies[name].add( (oper, version, cube) )

   515                     except Exception:

   516                         self.warnings.append(

   517                             'cube %s depends on %s but constraint badly '

   518                             'formatted: %s' % (cube, name, constraint))

   519                 else:

   520                     self.reverse_dependencies[name].add( (None, None, cube) )

   521         # check consistency

   522         for cube, versions in sorted(self.reverse_dependencies.items()):

   523             oper, version, source = None, None, None

   524             # simplify constraints

   525             if versions:

   526                 for constraint in versions:

   527                     op, ver, src = constraint

   528                     if oper is None:

   529                         oper = op

   530                         version = ver

   531                         source = src

   532                     elif op == '>=' and oper == '>=':

   533                         if version_strictly_lower(version, ver):

   534                             version = ver

   535                             source = src

   536                     elif op == None:

   537                         continue

   538                     else:

   539                         print('unable to handle %s in %s, set to `%s %s` '

   540                               'but currently up to `%s %s`' %

   541                               (cube, source, oper, version, op, ver))

   542             # "solve" constraint satisfaction problem

   543             if cube not in self.cubes:

   544                 self.errors.append( ('add', cube, version, source) )

   545             elif versions:

   546                 lower_strict = version_strictly_lower(self.cubes[cube], version)

   547                 if oper in ('>=','=','=='):

   548                     if lower_strict:

   549                         self.errors.append( ('update', cube, version, source) )

   550                 elif oper is None:

   551                     pass # no constraint on version

   552                 else:

   553                     print('unknown operator', oper)