1 """utilities for instances migration

     2 

     3 :organization: Logilab

     4 :copyright: 2001-2009 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2.

     5 :contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr

     6 :license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses

     7 """

     8 __docformat__ = "restructuredtext en"

     9 

    10 import sys

    11 import os

    12 import logging

    13 import tempfile

    14 from os.path import exists, join, basename, splitext

    15 

    16 from logilab.common.decorators import cached

    17 from logilab.common.configuration import REQUIRED, read_old_config

    18 from logilab.common.shellutils import ASK

    19 

    20 from cubicweb import ConfigurationError

    21 

    22 

    23 def filter_scripts(config, directory, fromversion, toversion, quiet=True):

    24     """return a list of paths of migration files to consider to upgrade

    25     from a version to a greater one

    26     """

    27     from logilab.common.changelog import Version # doesn't work with appengine

    28     assert fromversion

    29     assert toversion

    30     assert isinstance(fromversion, tuple), fromversion.__class__

    31     assert isinstance(toversion, tuple), toversion.__class__

    32     assert fromversion <= toversion, (fromversion, toversion)

    33     if not exists(directory):

    34         if not quiet:

    35             print directory, "doesn't exists, no migration path"

    36         return []

    37     if fromversion == toversion:

    38         return []

    39     result = []

    40     for fname in os.listdir(directory):

    41         if fname.endswith('.pyc') or fname.endswith('.pyo') \

    42                or fname.endswith('~'):

    43             continue

    44         fpath = join(directory, fname)

    45         try:

    46             tver, mode = fname.split('_', 1)

    47         except ValueError:

    48             continue

    49         mode = mode.split('.', 1)[0]

    50         if not config.accept_mode(mode):

    51             continue

    52         try:

    53             tver = Version(tver)

    54         except ValueError:

    55             continue

    56         if tver <= fromversion:

    57             continue

    58         if tver > toversion:

    59             continue

    60         result.append((tver, fpath))

    61     # be sure scripts are executed in order

    62     return sorted(result)

    63 

    64 

    65 IGNORED_EXTENSIONS = ('.swp', '~')

    66 

    67 

    68 def execscript_confirm(scriptpath):

    69     """asks for confirmation before executing a script and provides the

    70     ability to show the script's content

    71     """

    72     while True:

    73         answer = ASK.ask('Execute %r ?' % scriptpath, ('Y','n','show'), 'Y')

    74         if answer == 'n':

    75             return False

    76         elif answer == 'show':

    77             stream = open(scriptpath)

    78             scriptcontent = stream.read()

    79             stream.close()

    80             print

    81             print scriptcontent

    82             print

    83         else:

    84             return True

    85 

    86 def yes(*args, **kwargs):

    87     return True

    88 

    89 

    90 class MigrationHelper(object):

    91     """class holding CubicWeb Migration Actions used by migration scripts"""

    92 

    93     def __init__(self, config, interactive=True, verbosity=1):

    94         self.config = config

    95         if config:

    96             # no config on shell to a remote instance

    97             self.config.init_log(logthreshold=logging.ERROR, debug=True)

    98         # 0: no confirmation, 1: only main commands confirmed, 2 ask for everything

    99         self.verbosity = verbosity

   100         self.need_wrap = True

   101         if not interactive or not verbosity:

   102             self.confirm = yes

   103             self.execscript_confirm = yes

   104         else:

   105             self.execscript_confirm = execscript_confirm

   106         self._option_changes = []

   107         self.__context = {'confirm': self.confirm,

   108                           'config': self.config,

   109                           'interactive_mode': interactive,

   110                           }

   111 

   112     def __getattribute__(self, name):

   113         try:

   114             return object.__getattribute__(self, name)

   115         except AttributeError:

   116             cmd = 'cmd_%s' % name

   117             if hasattr(self, cmd):

   118                 meth = getattr(self, cmd)

   119                 return lambda *args, **kwargs: self.interact(args, kwargs,

   120                                                              meth=meth)

   121             raise

   122         raise AttributeError(name)

   123 

   124     def repo_connect(self):

   125         return self.config.repository()

   126 

   127     def migrate(self, vcconf, toupgrade, options):

   128         """upgrade the given set of cubes

   129 

   130         `cubes` is an ordered list of 3-uple:

   131         (cube, fromversion, toversion)

   132         """

   133         if options.fs_only:

   134             # monkey path configuration.accept_mode so database mode (e.g. Any)

   135             # won't be accepted

   136             orig_accept_mode = self.config.accept_mode

   137             def accept_mode(mode):

   138                 if mode == 'Any':

   139                     return False

   140                 return orig_accept_mode(mode)

   141             self.config.accept_mode = accept_mode

   142         # may be an iterator

   143         toupgrade = tuple(toupgrade)

   144         vmap = dict( (cube, (fromver, tover)) for cube, fromver, tover in toupgrade)

   145         ctx = self.__context

   146         ctx['versions_map'] = vmap

   147         if self.config.accept_mode('Any') and 'cubicweb' in vmap:

   148             migrdir = self.config.migration_scripts_dir()

   149             self.cmd_process_script(join(migrdir, 'bootstrapmigration_repository.py'))

   150         for cube, fromversion, toversion in toupgrade:

   151             if cube == 'cubicweb':

   152                 migrdir = self.config.migration_scripts_dir()

   153             else:

   154                 migrdir = self.config.cube_migration_scripts_dir(cube)

   155             scripts = filter_scripts(self.config, migrdir, fromversion, toversion)

   156             if scripts:

   157                 prevversion = None

   158                 for version, script in scripts:

   159                     # take care to X.Y.Z_Any.py / X.Y.Z_common.py: we've to call

   160                     # cube_upgraded once all script of X.Y.Z have been executed

   161                     if prevversion is not None and version != prevversion:

   162                         self.cube_upgraded(cube, prevversion)

   163                     prevversion = version

   164                     self.cmd_process_script(script)

   165                 self.cube_upgraded(cube, toversion)

   166             else:

   167                 self.cube_upgraded(cube, toversion)

   168 

   169     def cube_upgraded(self, cube, version):

   170         pass

   171 

   172     def shutdown(self):

   173         pass

   174 

   175     def interact(self, args, kwargs, meth):

   176         """execute the given method according to user's confirmation"""

   177         msg = 'Execute command: %s(%s) ?' % (

   178             meth.__name__[4:],

   179             ', '.join([repr(arg) for arg in args] +

   180                       ['%s=%r' % (n,v) for n,v in kwargs.items()]))

   181         if 'ask_confirm' in kwargs:

   182             ask_confirm = kwargs.pop('ask_confirm')

   183         else:

   184             ask_confirm = True

   185         if not ask_confirm or self.confirm(msg):

   186             return meth(*args, **kwargs)

   187 

   188     def confirm(self, question, shell=True, abort=True, retry=False, default='y'):

   189         """ask for confirmation and return true on positive answer

   190 

   191         if `retry` is true the r[etry] answer may return 2

   192         """

   193         possibleanswers = ['y','n']

   194         if abort:

   195             possibleanswers.append('abort')

   196         if shell:

   197             possibleanswers.append('shell')

   198         if retry:

   199             possibleanswers.append('retry')

   200         try:

   201             answer = ASK.ask(question, possibleanswers, default)

   202         except (EOFError, KeyboardInterrupt):

   203             answer = 'abort'

   204         if answer == 'n':

   205             return False

   206         if answer == 'retry':

   207             return 2

   208         if answer == 'abort':

   209             raise SystemExit(1)

   210         if shell and answer == 'shell':

   211             self.interactive_shell()

   212             return self.confirm(question)

   213         return True

   214 

   215     def interactive_shell(self):

   216         self.confirm = yes

   217         self.need_wrap = False

   218         # avoid '_' to be added to builtins by sys.display_hook

   219         def do_not_add___to_builtins(obj):

   220             if obj is not None:

   221                 print repr(obj)

   222         sys.displayhook = do_not_add___to_builtins

   223         local_ctx = self._create_context()

   224         try:

   225             import readline

   226             from rlcompleter import Completer

   227         except ImportError:

   228             # readline not available

   229             pass

   230         else:

   231             readline.set_completer(Completer(local_ctx).complete)

   232             readline.parse_and_bind('tab: complete')

   233             home_key = 'HOME'

   234             if sys.platform == 'win32':

   235                 home_key = 'USERPROFILE'

   236             histfile = os.path.join(os.environ[home_key], ".eshellhist")

   237             try:

   238                 readline.read_history_file(histfile)

   239             except IOError:

   240                 pass

   241         from code import interact

   242         banner = """entering the migration python shell

   243 just type migration commands or arbitrary python code and type ENTER to execute it

   244 type "exit" or Ctrl-D to quit the shell and resume operation"""

   245         # give custom readfunc to avoid http://bugs.python.org/issue1288615

   246         def unicode_raw_input(prompt):

   247             return unicode(raw_input(prompt), sys.stdin.encoding)

   248         interact(banner, readfunc=unicode_raw_input, local=local_ctx)

   249         readline.write_history_file(histfile)

   250         # delete instance's confirm attribute to avoid questions

   251         del self.confirm

   252         self.need_wrap = True

   253 

   254     @cached

   255     def _create_context(self):

   256         """return a dictionary to use as migration script execution context"""

   257         context = self.__context

   258         for attr in dir(self):

   259             if attr.startswith('cmd_'):

   260                 if self.need_wrap:

   261                     context[attr[4:]] = getattr(self, attr[4:])

   262                 else:

   263                     context[attr[4:]] = getattr(self, attr)

   264         return context

   265 

   266     def cmd_process_script(self, migrscript, funcname=None, *args, **kwargs):

   267         """execute a migration script

   268         in interactive mode,  display the migration script path, ask for

   269         confirmation and execute it if confirmed

   270         """

   271         migrscript = os.path.normpath(migrscript)

   272         if migrscript.endswith('.py'):

   273             script_mode = 'python'

   274         elif migrscript.endswith('.txt') or migrscript.endswith('.rst'):

   275             script_mode = 'doctest'

   276         else:

   277             raise Exception('This is not a valid cubicweb shell input')

   278         if not self.execscript_confirm(migrscript):

   279             return

   280         scriptlocals = self._create_context().copy()

   281         if script_mode == 'python':

   282             if funcname is None:

   283                 pyname = '__main__'

   284             else:

   285                 pyname = splitext(basename(migrscript))[0]

   286             scriptlocals.update({'__file__': migrscript, '__name__': pyname})

   287             execfile(migrscript, scriptlocals)

   288             if funcname is not None:

   289                 try:

   290                     func = scriptlocals[funcname]

   291                     self.info('found %s in locals', funcname)

   292                     assert callable(func), '%s (%s) is not callable' % (func, funcname)

   293                 except KeyError:

   294                     self.critical('no %s in script %s', funcname, migrscript)

   295                     return None

   296                 return func(*args, **kwargs)

   297         else: # script_mode == 'doctest'

   298             import doctest

   299             doctest.testfile(migrscript, module_relative=False,

   300                              optionflags=doctest.ELLIPSIS, globs=scriptlocals)

   301 

   302     def cmd_option_renamed(self, oldname, newname):

   303         """a configuration option has been renamed"""

   304         self._option_changes.append(('renamed', oldname, newname))

   305 

   306     def cmd_option_group_change(self, option, oldgroup, newgroup):

   307         """a configuration option has been moved in another group"""

   308         self._option_changes.append(('moved', option, oldgroup, newgroup))

   309 

   310     def cmd_option_added(self, optname):

   311         """a configuration option has been added"""

   312         self._option_changes.append(('added', optname))

   313 

   314     def cmd_option_removed(self, optname):

   315         """a configuration option has been removed"""

   316         # can safely be ignored

   317         #self._option_changes.append(('removed', optname))

   318 

   319     def cmd_option_type_changed(self, optname, oldtype, newvalue):

   320         """a configuration option's type has changed"""

   321         self._option_changes.append(('typechanged', optname, oldtype, newvalue))

   322 

   323     def cmd_add_cubes(self, cubes):

   324         """modify the list of used cubes in the in-memory config

   325         returns newly inserted cubes, including dependencies

   326         """

   327         if isinstance(cubes, basestring):

   328             cubes = (cubes,)

   329         origcubes = self.config.cubes()

   330         newcubes = [p for p in self.config.expand_cubes(cubes)

   331                        if not p in origcubes]

   332         if newcubes:

   333             for cube in cubes:

   334                 assert cube in newcubes

   335             self.config.add_cubes(newcubes)

   336         return newcubes

   337 

   338     def cmd_remove_cube(self, cube, removedeps=False):

   339         if removedeps:

   340             toremove = self.config.expand_cubes([cube])

   341         else:

   342             toremove = (cube,)

   343         origcubes = self.config._cubes

   344         basecubes = [c for c in origcubes if not c in toremove]

   345         self.config._cubes = tuple(self.config.expand_cubes(basecubes))

   346         removed = [p for p in origcubes if not p in self.config._cubes]

   347         if not cube in removed:

   348             raise ConfigurationError("can't remove cube %s, "

   349                                      "used as a dependency" % cube)

   350         return removed

   351 

   352     def rewrite_configuration(self):

   353         # import locally, show_diffs unavailable in gae environment

   354         from cubicweb.toolsutils import show_diffs

   355         configfile = self.config.main_config_file()

   356         if self._option_changes:

   357             read_old_config(self.config, self._option_changes, configfile)

   358         fd, newconfig = tempfile.mkstemp()

   359         for optdescr in self._option_changes:

   360             if optdescr[0] == 'added':

   361                 optdict = self.config.get_option_def(optdescr[1])

   362                 if optdict.get('default') is REQUIRED:

   363                     self.config.input_option(optdescr[1], optdict)

   364         self.config.generate_config(open(newconfig, 'w'))

   365         show_diffs(configfile, newconfig)

   366         os.close(fd)

   367         if exists(newconfig):

   368             os.unlink(newconfig)

   369 

   370 

   371 from logging import getLogger

   372 from cubicweb import set_log_methods

   373 set_log_methods(MigrationHelper, getLogger('cubicweb.migration'))