migration.py
changeset 4021 280c910c8710
parent 3998 94cc7cad3d2d
child 4252 6c4f109c2b03
equal deleted inserted replaced
4018:d4d4e7112ccf 4021:280c910c8710
       
     1 """utilities for instances migration
       
     2 
       
     3 :organization: Logilab
       
     4 :copyright: 2001-2009 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2.
       
     5 :contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr
       
     6 :license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses
       
     7 """
       
     8 __docformat__ = "restructuredtext en"
       
     9 
       
    10 import sys
       
    11 import os
       
    12 import logging
       
    13 import tempfile
       
    14 from os.path import exists, join, basename, splitext
       
    15 
       
    16 from logilab.common.decorators import cached
       
    17 from logilab.common.configuration import REQUIRED, read_old_config
       
    18 from logilab.common.shellutils import ASK
       
    19 
       
    20 from cubicweb import ConfigurationError
       
    21 
       
    22 
       
    23 def filter_scripts(config, directory, fromversion, toversion, quiet=True):
       
    24     """return a list of paths of migration files to consider to upgrade
       
    25     from a version to a greater one
       
    26     """
       
    27     from logilab.common.changelog import Version # doesn't work with appengine
       
    28     assert fromversion
       
    29     assert toversion
       
    30     assert isinstance(fromversion, tuple), fromversion.__class__
       
    31     assert isinstance(toversion, tuple), toversion.__class__
       
    32     assert fromversion <= toversion, (fromversion, toversion)
       
    33     if not exists(directory):
       
    34         if not quiet:
       
    35             print directory, "doesn't exists, no migration path"
       
    36         return []
       
    37     if fromversion == toversion:
       
    38         return []
       
    39     result = []
       
    40     for fname in os.listdir(directory):
       
    41         if fname.endswith('.pyc') or fname.endswith('.pyo') \
       
    42                or fname.endswith('~'):
       
    43             continue
       
    44         fpath = join(directory, fname)
       
    45         try:
       
    46             tver, mode = fname.split('_', 1)
       
    47         except ValueError:
       
    48             continue
       
    49         mode = mode.split('.', 1)[0]
       
    50         if not config.accept_mode(mode):
       
    51             continue
       
    52         try:
       
    53             tver = Version(tver)
       
    54         except ValueError:
       
    55             continue
       
    56         if tver <= fromversion:
       
    57             continue
       
    58         if tver > toversion:
       
    59             continue
       
    60         result.append((tver, fpath))
       
    61     # be sure scripts are executed in order
       
    62     return sorted(result)
       
    63 
       
    64 
       
    65 IGNORED_EXTENSIONS = ('.swp', '~')
       
    66 
       
    67 
       
    68 def execscript_confirm(scriptpath):
       
    69     """asks for confirmation before executing a script and provides the
       
    70     ability to show the script's content
       
    71     """
       
    72     while True:
       
    73         answer = ASK.ask('Execute %r ?' % scriptpath, ('Y','n','show'), 'Y')
       
    74         if answer == 'n':
       
    75             return False
       
    76         elif answer == 'show':
       
    77             stream = open(scriptpath)
       
    78             scriptcontent = stream.read()
       
    79             stream.close()
       
    80             print
       
    81             print scriptcontent
       
    82             print
       
    83         else:
       
    84             return True
       
    85 
       
    86 def yes(*args, **kwargs):
       
    87     return True
       
    88 
       
    89 
       
    90 class MigrationHelper(object):
       
    91     """class holding CubicWeb Migration Actions used by migration scripts"""
       
    92 
       
    93     def __init__(self, config, interactive=True, verbosity=1):
       
    94         self.config = config
       
    95         if config:
       
    96             # no config on shell to a remote instance
       
    97             self.config.init_log(logthreshold=logging.ERROR, debug=True)
       
    98         # 0: no confirmation, 1: only main commands confirmed, 2 ask for everything
       
    99         self.verbosity = verbosity
       
   100         self.need_wrap = True
       
   101         if not interactive or not verbosity:
       
   102             self.confirm = yes
       
   103             self.execscript_confirm = yes
       
   104         else:
       
   105             self.execscript_confirm = execscript_confirm
       
   106         self._option_changes = []
       
   107         self.__context = {'confirm': self.confirm,
       
   108                           'config': self.config,
       
   109                           'interactive_mode': interactive,
       
   110                           }
       
   111 
       
   112     def __getattribute__(self, name):
       
   113         try:
       
   114             return object.__getattribute__(self, name)
       
   115         except AttributeError:
       
   116             cmd = 'cmd_%s' % name
       
   117             if hasattr(self, cmd):
       
   118                 meth = getattr(self, cmd)
       
   119                 return lambda *args, **kwargs: self.interact(args, kwargs,
       
   120                                                              meth=meth)
       
   121             raise
       
   122         raise AttributeError(name)
       
   123 
       
   124     def repo_connect(self):
       
   125         return self.config.repository()
       
   126 
       
   127     def migrate(self, vcconf, toupgrade, options):
       
   128         """upgrade the given set of cubes
       
   129 
       
   130         `cubes` is an ordered list of 3-uple:
       
   131         (cube, fromversion, toversion)
       
   132         """
       
   133         if options.fs_only:
       
   134             # monkey path configuration.accept_mode so database mode (e.g. Any)
       
   135             # won't be accepted
       
   136             orig_accept_mode = self.config.accept_mode
       
   137             def accept_mode(mode):
       
   138                 if mode == 'Any':
       
   139                     return False
       
   140                 return orig_accept_mode(mode)
       
   141             self.config.accept_mode = accept_mode
       
   142         # may be an iterator
       
   143         toupgrade = tuple(toupgrade)
       
   144         vmap = dict( (cube, (fromver, tover)) for cube, fromver, tover in toupgrade)
       
   145         ctx = self.__context
       
   146         ctx['versions_map'] = vmap
       
   147         if self.config.accept_mode('Any') and 'cubicweb' in vmap:
       
   148             migrdir = self.config.migration_scripts_dir()
       
   149             self.cmd_process_script(join(migrdir, 'bootstrapmigration_repository.py'))
       
   150         for cube, fromversion, toversion in toupgrade:
       
   151             if cube == 'cubicweb':
       
   152                 migrdir = self.config.migration_scripts_dir()
       
   153             else:
       
   154                 migrdir = self.config.cube_migration_scripts_dir(cube)
       
   155             scripts = filter_scripts(self.config, migrdir, fromversion, toversion)
       
   156             if scripts:
       
   157                 prevversion = None
       
   158                 for version, script in scripts:
       
   159                     # take care to X.Y.Z_Any.py / X.Y.Z_common.py: we've to call
       
   160                     # cube_upgraded once all script of X.Y.Z have been executed
       
   161                     if prevversion is not None and version != prevversion:
       
   162                         self.cube_upgraded(cube, prevversion)
       
   163                     prevversion = version
       
   164                     self.cmd_process_script(script)
       
   165                 self.cube_upgraded(cube, toversion)
       
   166             else:
       
   167                 self.cube_upgraded(cube, toversion)
       
   168 
       
   169     def cube_upgraded(self, cube, version):
       
   170         pass
       
   171 
       
   172     def shutdown(self):
       
   173         pass
       
   174 
       
   175     def interact(self, args, kwargs, meth):
       
   176         """execute the given method according to user's confirmation"""
       
   177         msg = 'Execute command: %s(%s) ?' % (
       
   178             meth.__name__[4:],
       
   179             ', '.join([repr(arg) for arg in args] +
       
   180                       ['%s=%r' % (n,v) for n,v in kwargs.items()]))
       
   181         if 'ask_confirm' in kwargs:
       
   182             ask_confirm = kwargs.pop('ask_confirm')
       
   183         else:
       
   184             ask_confirm = True
       
   185         if not ask_confirm or self.confirm(msg):
       
   186             return meth(*args, **kwargs)
       
   187 
       
   188     def confirm(self, question, shell=True, abort=True, retry=False, default='y'):
       
   189         """ask for confirmation and return true on positive answer
       
   190 
       
   191         if `retry` is true the r[etry] answer may return 2
       
   192         """
       
   193         possibleanswers = ['y','n']
       
   194         if abort:
       
   195             possibleanswers.append('abort')
       
   196         if shell:
       
   197             possibleanswers.append('shell')
       
   198         if retry:
       
   199             possibleanswers.append('retry')
       
   200         try:
       
   201             answer = ASK.ask(question, possibleanswers, default)
       
   202         except (EOFError, KeyboardInterrupt):
       
   203             answer = 'abort'
       
   204         if answer == 'n':
       
   205             return False
       
   206         if answer == 'retry':
       
   207             return 2
       
   208         if answer == 'abort':
       
   209             raise SystemExit(1)
       
   210         if shell and answer == 'shell':
       
   211             self.interactive_shell()
       
   212             return self.confirm(question)
       
   213         return True
       
   214 
       
   215     def interactive_shell(self):
       
   216         self.confirm = yes
       
   217         self.need_wrap = False
       
   218         # avoid '_' to be added to builtins by sys.display_hook
       
   219         def do_not_add___to_builtins(obj):
       
   220             if obj is not None:
       
   221                 print repr(obj)
       
   222         sys.displayhook = do_not_add___to_builtins
       
   223         local_ctx = self._create_context()
       
   224         try:
       
   225             import readline
       
   226             from rlcompleter import Completer
       
   227         except ImportError:
       
   228             # readline not available
       
   229             pass
       
   230         else:
       
   231             readline.set_completer(Completer(local_ctx).complete)
       
   232             readline.parse_and_bind('tab: complete')
       
   233             home_key = 'HOME'
       
   234             if sys.platform == 'win32':
       
   235                 home_key = 'USERPROFILE'
       
   236             histfile = os.path.join(os.environ[home_key], ".eshellhist")
       
   237             try:
       
   238                 readline.read_history_file(histfile)
       
   239             except IOError:
       
   240                 pass
       
   241         from code import interact
       
   242         banner = """entering the migration python shell
       
   243 just type migration commands or arbitrary python code and type ENTER to execute it
       
   244 type "exit" or Ctrl-D to quit the shell and resume operation"""
       
   245         # give custom readfunc to avoid http://bugs.python.org/issue1288615
       
   246         def unicode_raw_input(prompt):
       
   247             return unicode(raw_input(prompt), sys.stdin.encoding)
       
   248         interact(banner, readfunc=unicode_raw_input, local=local_ctx)
       
   249         readline.write_history_file(histfile)
       
   250         # delete instance's confirm attribute to avoid questions
       
   251         del self.confirm
       
   252         self.need_wrap = True
       
   253 
       
   254     @cached
       
   255     def _create_context(self):
       
   256         """return a dictionary to use as migration script execution context"""
       
   257         context = self.__context
       
   258         for attr in dir(self):
       
   259             if attr.startswith('cmd_'):
       
   260                 if self.need_wrap:
       
   261                     context[attr[4:]] = getattr(self, attr[4:])
       
   262                 else:
       
   263                     context[attr[4:]] = getattr(self, attr)
       
   264         return context
       
   265 
       
   266     def cmd_process_script(self, migrscript, funcname=None, *args, **kwargs):
       
   267         """execute a migration script
       
   268         in interactive mode,  display the migration script path, ask for
       
   269         confirmation and execute it if confirmed
       
   270         """
       
   271         migrscript = os.path.normpath(migrscript)
       
   272         if migrscript.endswith('.py'):
       
   273             script_mode = 'python'
       
   274         elif migrscript.endswith('.txt') or migrscript.endswith('.rst'):
       
   275             script_mode = 'doctest'
       
   276         else:
       
   277             raise Exception('This is not a valid cubicweb shell input')
       
   278         if not self.execscript_confirm(migrscript):
       
   279             return
       
   280         scriptlocals = self._create_context().copy()
       
   281         if script_mode == 'python':
       
   282             if funcname is None:
       
   283                 pyname = '__main__'
       
   284             else:
       
   285                 pyname = splitext(basename(migrscript))[0]
       
   286             scriptlocals.update({'__file__': migrscript, '__name__': pyname})
       
   287             execfile(migrscript, scriptlocals)
       
   288             if funcname is not None:
       
   289                 try:
       
   290                     func = scriptlocals[funcname]
       
   291                     self.info('found %s in locals', funcname)
       
   292                     assert callable(func), '%s (%s) is not callable' % (func, funcname)
       
   293                 except KeyError:
       
   294                     self.critical('no %s in script %s', funcname, migrscript)
       
   295                     return None
       
   296                 return func(*args, **kwargs)
       
   297         else: # script_mode == 'doctest'
       
   298             import doctest
       
   299             doctest.testfile(migrscript, module_relative=False,
       
   300                              optionflags=doctest.ELLIPSIS, globs=scriptlocals)
       
   301 
       
   302     def cmd_option_renamed(self, oldname, newname):
       
   303         """a configuration option has been renamed"""
       
   304         self._option_changes.append(('renamed', oldname, newname))
       
   305 
       
   306     def cmd_option_group_change(self, option, oldgroup, newgroup):
       
   307         """a configuration option has been moved in another group"""
       
   308         self._option_changes.append(('moved', option, oldgroup, newgroup))
       
   309 
       
   310     def cmd_option_added(self, optname):
       
   311         """a configuration option has been added"""
       
   312         self._option_changes.append(('added', optname))
       
   313 
       
   314     def cmd_option_removed(self, optname):
       
   315         """a configuration option has been removed"""
       
   316         # can safely be ignored
       
   317         #self._option_changes.append(('removed', optname))
       
   318 
       
   319     def cmd_option_type_changed(self, optname, oldtype, newvalue):
       
   320         """a configuration option's type has changed"""
       
   321         self._option_changes.append(('typechanged', optname, oldtype, newvalue))
       
   322 
       
   323     def cmd_add_cubes(self, cubes):
       
   324         """modify the list of used cubes in the in-memory config
       
   325         returns newly inserted cubes, including dependencies
       
   326         """
       
   327         if isinstance(cubes, basestring):
       
   328             cubes = (cubes,)
       
   329         origcubes = self.config.cubes()
       
   330         newcubes = [p for p in self.config.expand_cubes(cubes)
       
   331                        if not p in origcubes]
       
   332         if newcubes:
       
   333             for cube in cubes:
       
   334                 assert cube in newcubes
       
   335             self.config.add_cubes(newcubes)
       
   336         return newcubes
       
   337 
       
   338     def cmd_remove_cube(self, cube, removedeps=False):
       
   339         if removedeps:
       
   340             toremove = self.config.expand_cubes([cube])
       
   341         else:
       
   342             toremove = (cube,)
       
   343         origcubes = self.config._cubes
       
   344         basecubes = [c for c in origcubes if not c in toremove]
       
   345         self.config._cubes = tuple(self.config.expand_cubes(basecubes))
       
   346         removed = [p for p in origcubes if not p in self.config._cubes]
       
   347         if not cube in removed:
       
   348             raise ConfigurationError("can't remove cube %s, "
       
   349                                      "used as a dependency" % cube)
       
   350         return removed
       
   351 
       
   352     def rewrite_configuration(self):
       
   353         # import locally, show_diffs unavailable in gae environment
       
   354         from cubicweb.toolsutils import show_diffs
       
   355         configfile = self.config.main_config_file()
       
   356         if self._option_changes:
       
   357             read_old_config(self.config, self._option_changes, configfile)
       
   358         fd, newconfig = tempfile.mkstemp()
       
   359         for optdescr in self._option_changes:
       
   360             if optdescr[0] == 'added':
       
   361                 optdict = self.config.get_option_def(optdescr[1])
       
   362                 if optdict.get('default') is REQUIRED:
       
   363                     self.config.input_option(optdescr[1], optdict)
       
   364         self.config.generate_config(open(newconfig, 'w'))
       
   365         show_diffs(configfile, newconfig)
       
   366         os.close(fd)
       
   367         if exists(newconfig):
       
   368             os.unlink(newconfig)
       
   369 
       
   370 
       
   371 from logging import getLogger
       
   372 from cubicweb import set_log_methods
       
   373 set_log_methods(MigrationHelper, getLogger('cubicweb.migration'))