|
1 # copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved. |
|
2 # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr |
|
3 # |
|
4 # This file is part of CubicWeb. |
|
5 # |
|
6 # CubicWeb is free software: you can redistribute it and/or modify it under the |
|
7 # terms of the GNU Lesser General Public License as published by the Free |
|
8 # Software Foundation, either version 2.1 of the License, or (at your option) |
|
9 # any later version. |
|
10 # |
|
11 # CubicWeb is distributed in the hope that it will be useful, but WITHOUT |
|
12 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS |
|
13 # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more |
|
14 # details. |
|
15 # |
|
16 # You should have received a copy of the GNU Lesser General Public License along |
|
17 # with CubicWeb. If not, see <http://www.gnu.org/licenses/>. |
|
18 """utilities for instances migration""" |
|
19 from __future__ import print_function |
|
20 |
|
21 __docformat__ = "restructuredtext en" |
|
22 |
|
23 import sys |
|
24 import os |
|
25 import logging |
|
26 import tempfile |
|
27 from os.path import exists, join, basename, splitext |
|
28 from itertools import chain |
|
29 from warnings import warn |
|
30 |
|
31 from six import string_types |
|
32 |
|
33 from logilab.common import IGNORED_EXTENSIONS |
|
34 from logilab.common.decorators import cached |
|
35 from logilab.common.configuration import REQUIRED, read_old_config |
|
36 from logilab.common.shellutils import ASK |
|
37 from logilab.common.changelog import Version |
|
38 from logilab.common.deprecation import deprecated |
|
39 |
|
40 from cubicweb import ConfigurationError, ExecutionError |
|
41 from cubicweb.cwconfig import CubicWebConfiguration as cwcfg |
|
42 from cubicweb.toolsutils import show_diffs |
|
43 |
|
44 def filter_scripts(config, directory, fromversion, toversion, quiet=True): |
|
45 """return a list of paths of migration files to consider to upgrade |
|
46 from a version to a greater one |
|
47 """ |
|
48 from logilab.common.changelog import Version # doesn't work with appengine |
|
49 assert fromversion |
|
50 assert toversion |
|
51 assert isinstance(fromversion, tuple), fromversion.__class__ |
|
52 assert isinstance(toversion, tuple), toversion.__class__ |
|
53 assert fromversion <= toversion, (fromversion, toversion) |
|
54 if not exists(directory): |
|
55 if not quiet: |
|
56 print(directory, "doesn't exists, no migration path") |
|
57 return [] |
|
58 if fromversion == toversion: |
|
59 return [] |
|
60 result = [] |
|
61 for fname in os.listdir(directory): |
|
62 if fname.endswith(IGNORED_EXTENSIONS): |
|
63 continue |
|
64 fpath = join(directory, fname) |
|
65 try: |
|
66 tver, mode = fname.split('_', 1) |
|
67 except ValueError: |
|
68 continue |
|
69 mode = mode.split('.', 1)[0] |
|
70 if not config.accept_mode(mode): |
|
71 continue |
|
72 try: |
|
73 tver = Version(tver) |
|
74 except ValueError: |
|
75 continue |
|
76 if tver <= fromversion: |
|
77 continue |
|
78 if tver > toversion: |
|
79 continue |
|
80 result.append((tver, fpath)) |
|
81 # be sure scripts are executed in order |
|
82 return sorted(result) |
|
83 |
|
84 |
|
85 def execscript_confirm(scriptpath): |
|
86 """asks for confirmation before executing a script and provides the |
|
87 ability to show the script's content |
|
88 """ |
|
89 while True: |
|
90 answer = ASK.ask('Execute %r ?' % scriptpath, |
|
91 ('Y','n','show','abort'), 'Y') |
|
92 if answer == 'abort': |
|
93 raise SystemExit(1) |
|
94 elif answer == 'n': |
|
95 return False |
|
96 elif answer == 'show': |
|
97 stream = open(scriptpath) |
|
98 scriptcontent = stream.read() |
|
99 stream.close() |
|
100 print() |
|
101 print(scriptcontent) |
|
102 print() |
|
103 else: |
|
104 return True |
|
105 |
|
106 def yes(*args, **kwargs): |
|
107 return True |
|
108 |
|
109 |
|
110 class MigrationHelper(object): |
|
111 """class holding CubicWeb Migration Actions used by migration scripts""" |
|
112 |
|
113 def __init__(self, config, interactive=True, verbosity=1): |
|
114 self.config = config |
|
115 if config: |
|
116 # no config on shell to a remote instance |
|
117 self.config.init_log(logthreshold=logging.ERROR) |
|
118 # 0: no confirmation, 1: only main commands confirmed, 2 ask for everything |
|
119 self.verbosity = verbosity |
|
120 self.need_wrap = True |
|
121 if not interactive or not verbosity: |
|
122 self.confirm = yes |
|
123 self.execscript_confirm = yes |
|
124 else: |
|
125 self.execscript_confirm = execscript_confirm |
|
126 self._option_changes = [] |
|
127 self.__context = {'confirm': self.confirm, |
|
128 'config': self.config, |
|
129 'interactive_mode': interactive, |
|
130 } |
|
131 self._context_stack = [] |
|
132 |
|
133 def __getattribute__(self, name): |
|
134 try: |
|
135 return object.__getattribute__(self, name) |
|
136 except AttributeError: |
|
137 cmd = 'cmd_%s' % name |
|
138 # search self.__class__ to avoid infinite recursion |
|
139 if hasattr(self.__class__, cmd): |
|
140 meth = getattr(self, cmd) |
|
141 return lambda *args, **kwargs: self.interact(args, kwargs, |
|
142 meth=meth) |
|
143 raise |
|
144 raise AttributeError(name) |
|
145 |
|
146 def migrate(self, vcconf, toupgrade, options): |
|
147 """upgrade the given set of cubes |
|
148 |
|
149 `cubes` is an ordered list of 3-uple: |
|
150 (cube, fromversion, toversion) |
|
151 """ |
|
152 if options.fs_only: |
|
153 # monkey path configuration.accept_mode so database mode (e.g. Any) |
|
154 # won't be accepted |
|
155 orig_accept_mode = self.config.accept_mode |
|
156 def accept_mode(mode): |
|
157 if mode == 'Any': |
|
158 return False |
|
159 return orig_accept_mode(mode) |
|
160 self.config.accept_mode = accept_mode |
|
161 # may be an iterator |
|
162 toupgrade = tuple(toupgrade) |
|
163 vmap = dict( (cube, (fromver, tover)) for cube, fromver, tover in toupgrade) |
|
164 ctx = self.__context |
|
165 ctx['versions_map'] = vmap |
|
166 if self.config.accept_mode('Any') and 'cubicweb' in vmap: |
|
167 migrdir = self.config.migration_scripts_dir() |
|
168 self.cmd_process_script(join(migrdir, 'bootstrapmigration_repository.py')) |
|
169 for cube, fromversion, toversion in toupgrade: |
|
170 if cube == 'cubicweb': |
|
171 migrdir = self.config.migration_scripts_dir() |
|
172 else: |
|
173 migrdir = self.config.cube_migration_scripts_dir(cube) |
|
174 scripts = filter_scripts(self.config, migrdir, fromversion, toversion) |
|
175 if scripts: |
|
176 prevversion = None |
|
177 for version, script in scripts: |
|
178 # take care to X.Y.Z_Any.py / X.Y.Z_common.py: we've to call |
|
179 # cube_upgraded once all script of X.Y.Z have been executed |
|
180 if prevversion is not None and version != prevversion: |
|
181 self.cube_upgraded(cube, prevversion) |
|
182 prevversion = version |
|
183 self.cmd_process_script(script) |
|
184 self.cube_upgraded(cube, toversion) |
|
185 else: |
|
186 self.cube_upgraded(cube, toversion) |
|
187 |
|
188 def cube_upgraded(self, cube, version): |
|
189 pass |
|
190 |
|
191 def shutdown(self): |
|
192 pass |
|
193 |
|
194 def interact(self, args, kwargs, meth): |
|
195 """execute the given method according to user's confirmation""" |
|
196 msg = 'Execute command: %s(%s) ?' % ( |
|
197 meth.__name__[4:], |
|
198 ', '.join([repr(arg) for arg in args] + |
|
199 ['%s=%r' % (n,v) for n,v in kwargs.items()])) |
|
200 if 'ask_confirm' in kwargs: |
|
201 ask_confirm = kwargs.pop('ask_confirm') |
|
202 else: |
|
203 ask_confirm = True |
|
204 if not ask_confirm or self.confirm(msg): |
|
205 return meth(*args, **kwargs) |
|
206 |
|
207 def confirm(self, question, # pylint: disable=E0202 |
|
208 shell=True, abort=True, retry=False, pdb=False, default='y'): |
|
209 """ask for confirmation and return true on positive answer |
|
210 |
|
211 if `retry` is true the r[etry] answer may return 2 |
|
212 """ |
|
213 possibleanswers = ['y', 'n'] |
|
214 if abort: |
|
215 possibleanswers.append('abort') |
|
216 if pdb: |
|
217 possibleanswers.append('pdb') |
|
218 if shell: |
|
219 possibleanswers.append('shell') |
|
220 if retry: |
|
221 possibleanswers.append('retry') |
|
222 try: |
|
223 answer = ASK.ask(question, possibleanswers, default) |
|
224 except (EOFError, KeyboardInterrupt): |
|
225 answer = 'abort' |
|
226 if answer == 'n': |
|
227 return False |
|
228 if answer == 'retry': |
|
229 return 2 |
|
230 if answer == 'abort': |
|
231 raise SystemExit(1) |
|
232 if answer == 'shell': |
|
233 self.interactive_shell() |
|
234 return self.confirm(question, shell, abort, retry, pdb, default) |
|
235 if answer == 'pdb': |
|
236 import pdb |
|
237 pdb.set_trace() |
|
238 return self.confirm(question, shell, abort, retry, pdb, default) |
|
239 return True |
|
240 |
|
241 def interactive_shell(self): |
|
242 self.confirm = yes |
|
243 self.need_wrap = False |
|
244 # avoid '_' to be added to builtins by sys.display_hook |
|
245 def do_not_add___to_builtins(obj): |
|
246 if obj is not None: |
|
247 print(repr(obj)) |
|
248 sys.displayhook = do_not_add___to_builtins |
|
249 local_ctx = self._create_context() |
|
250 try: |
|
251 import readline |
|
252 from cubicweb.toolsutils import CWShellCompleter |
|
253 except ImportError: |
|
254 # readline not available |
|
255 pass |
|
256 else: |
|
257 rql_completer = CWShellCompleter(local_ctx) |
|
258 readline.set_completer(rql_completer.complete) |
|
259 readline.parse_and_bind('tab: complete') |
|
260 home_key = 'HOME' |
|
261 if sys.platform == 'win32': |
|
262 home_key = 'USERPROFILE' |
|
263 histfile = os.path.join(os.environ[home_key], ".cwshell_history") |
|
264 try: |
|
265 readline.read_history_file(histfile) |
|
266 except IOError: |
|
267 pass |
|
268 from code import interact |
|
269 banner = """entering the migration python shell |
|
270 just type migration commands or arbitrary python code and type ENTER to execute it |
|
271 type "exit" or Ctrl-D to quit the shell and resume operation""" |
|
272 # give custom readfunc to avoid http://bugs.python.org/issue1288615 |
|
273 def unicode_raw_input(prompt): |
|
274 return unicode(raw_input(prompt), sys.stdin.encoding) |
|
275 interact(banner, readfunc=unicode_raw_input, local=local_ctx) |
|
276 try: |
|
277 readline.write_history_file(histfile) |
|
278 except IOError: |
|
279 pass |
|
280 # delete instance's confirm attribute to avoid questions |
|
281 del self.confirm |
|
282 self.need_wrap = True |
|
283 |
|
284 @cached |
|
285 def _create_context(self): |
|
286 """return a dictionary to use as migration script execution context""" |
|
287 context = self.__context |
|
288 for attr in dir(self): |
|
289 if attr.startswith('cmd_'): |
|
290 if self.need_wrap: |
|
291 context[attr[4:]] = getattr(self, attr[4:]) |
|
292 else: |
|
293 context[attr[4:]] = getattr(self, attr) |
|
294 return context |
|
295 |
|
296 def update_context(self, key, value): |
|
297 for context in self._context_stack: |
|
298 context[key] = value |
|
299 self.__context[key] = value |
|
300 |
|
301 def cmd_process_script(self, migrscript, funcname=None, *args, **kwargs): |
|
302 """execute a migration script in interactive mode |
|
303 |
|
304 Display the migration script path, ask for confirmation and execute it |
|
305 if confirmed |
|
306 |
|
307 Allowed input file formats for migration scripts: |
|
308 - `python` (.py) |
|
309 - `sql` (.sql) |
|
310 - `doctest` (.txt or .rst) |
|
311 |
|
312 .. warning:: sql migration scripts are not available in web-only instance |
|
313 |
|
314 You can pass script parameters with using double dash (--) in the |
|
315 command line |
|
316 |
|
317 Context environment can have these variables defined: |
|
318 - __name__ : will be determine by funcname parameter |
|
319 - __file__ : is the name of the script if it exists |
|
320 - __args__ : script arguments coming from command-line |
|
321 |
|
322 :param migrscript: name of the script |
|
323 :param funcname: defines __name__ inside the shell (or use __main__) |
|
324 :params args: optional arguments for funcname |
|
325 :keyword scriptargs: optional arguments of the script |
|
326 """ |
|
327 ftypes = {'python': ('.py',), |
|
328 'doctest': ('.txt', '.rst'), |
|
329 'sql': ('.sql',)} |
|
330 # sql migration scripts are not available in web-only instance |
|
331 if not hasattr(self, "session"): |
|
332 ftypes.pop('sql') |
|
333 migrscript = os.path.normpath(migrscript) |
|
334 for (script_mode, ftype) in ftypes.items(): |
|
335 if migrscript.endswith(ftype): |
|
336 break |
|
337 else: |
|
338 ftypes = ', '.join(chain(*ftypes.values())) |
|
339 msg = 'ignoring %s, not a valid script extension (%s)' |
|
340 raise ExecutionError(msg % (migrscript, ftypes)) |
|
341 if not self.execscript_confirm(migrscript): |
|
342 return |
|
343 scriptlocals = self._create_context().copy() |
|
344 scriptlocals.update({'__file__': migrscript, |
|
345 '__args__': kwargs.pop("scriptargs", [])}) |
|
346 self._context_stack.append(scriptlocals) |
|
347 if script_mode == 'python': |
|
348 if funcname is None: |
|
349 pyname = '__main__' |
|
350 else: |
|
351 pyname = splitext(basename(migrscript))[0] |
|
352 scriptlocals['__name__'] = pyname |
|
353 with open(migrscript, 'rb') as fobj: |
|
354 fcontent = fobj.read() |
|
355 try: |
|
356 code = compile(fcontent, migrscript, 'exec') |
|
357 except SyntaxError: |
|
358 # try without print_function |
|
359 code = compile(fcontent, migrscript, 'exec', 0, True) |
|
360 warn('[3.22] script %r should be updated to work with print_function' |
|
361 % migrscript, DeprecationWarning) |
|
362 exec(code, scriptlocals) |
|
363 if funcname is not None: |
|
364 try: |
|
365 func = scriptlocals[funcname] |
|
366 self.info('found %s in locals', funcname) |
|
367 assert callable(func), '%s (%s) is not callable' % (func, funcname) |
|
368 except KeyError: |
|
369 self.critical('no %s in script %s', funcname, migrscript) |
|
370 return None |
|
371 return func(*args, **kwargs) |
|
372 elif script_mode == 'sql': |
|
373 from cubicweb.server.sqlutils import sqlexec |
|
374 sqlexec(open(migrscript).read(), self.session.system_sql) |
|
375 self.commit() |
|
376 else: # script_mode == 'doctest' |
|
377 import doctest |
|
378 return doctest.testfile(migrscript, module_relative=False, |
|
379 optionflags=doctest.ELLIPSIS, |
|
380 # verbose mode when user input is expected |
|
381 verbose=self.verbosity==2, |
|
382 report=True, |
|
383 encoding='utf-8', |
|
384 globs=scriptlocals) |
|
385 self._context_stack.pop() |
|
386 |
|
387 def cmd_option_renamed(self, oldname, newname): |
|
388 """a configuration option has been renamed""" |
|
389 self._option_changes.append(('renamed', oldname, newname)) |
|
390 |
|
391 def cmd_option_group_changed(self, option, oldgroup, newgroup): |
|
392 """a configuration option has been moved in another group""" |
|
393 self._option_changes.append(('moved', option, oldgroup, newgroup)) |
|
394 |
|
395 def cmd_option_added(self, optname): |
|
396 """a configuration option has been added""" |
|
397 self._option_changes.append(('added', optname)) |
|
398 |
|
399 def cmd_option_removed(self, optname): |
|
400 """a configuration option has been removed""" |
|
401 # can safely be ignored |
|
402 #self._option_changes.append(('removed', optname)) |
|
403 |
|
404 def cmd_option_type_changed(self, optname, oldtype, newvalue): |
|
405 """a configuration option's type has changed""" |
|
406 self._option_changes.append(('typechanged', optname, oldtype, newvalue)) |
|
407 |
|
408 def cmd_add_cubes(self, cubes): |
|
409 """modify the list of used cubes in the in-memory config |
|
410 returns newly inserted cubes, including dependencies |
|
411 """ |
|
412 if isinstance(cubes, string_types): |
|
413 cubes = (cubes,) |
|
414 origcubes = self.config.cubes() |
|
415 newcubes = [p for p in self.config.expand_cubes(cubes) |
|
416 if not p in origcubes] |
|
417 if newcubes: |
|
418 self.config.add_cubes(newcubes) |
|
419 return newcubes |
|
420 |
|
421 @deprecated('[3.20] use drop_cube() instead of remove_cube()') |
|
422 def cmd_remove_cube(self, cube, removedeps=False): |
|
423 return self.cmd_drop_cube(cube, removedeps) |
|
424 |
|
425 def cmd_drop_cube(self, cube, removedeps=False): |
|
426 if removedeps: |
|
427 toremove = self.config.expand_cubes([cube]) |
|
428 else: |
|
429 toremove = (cube,) |
|
430 origcubes = self.config._cubes |
|
431 basecubes = [c for c in origcubes if not c in toremove] |
|
432 # don't fake-add any new ones, or we won't be able to really-add them later |
|
433 self.config._cubes = tuple(cube for cube in self.config.expand_cubes(basecubes) |
|
434 if cube in origcubes) |
|
435 removed = [p for p in origcubes if not p in self.config._cubes] |
|
436 if not cube in removed and cube in origcubes: |
|
437 raise ConfigurationError("can't remove cube %s, " |
|
438 "used as a dependency" % cube) |
|
439 return removed |
|
440 |
|
441 def rewrite_configuration(self): |
|
442 configfile = self.config.main_config_file() |
|
443 if self._option_changes: |
|
444 read_old_config(self.config, self._option_changes, configfile) |
|
445 fd, newconfig = tempfile.mkstemp() |
|
446 for optdescr in self._option_changes: |
|
447 if optdescr[0] == 'added': |
|
448 optdict = self.config.get_option_def(optdescr[1]) |
|
449 if optdict.get('default') is REQUIRED: |
|
450 self.config.input_option(optdescr[1], optdict) |
|
451 self.config.generate_config(open(newconfig, 'w')) |
|
452 show_diffs(configfile, newconfig, askconfirm=self.confirm is not yes) |
|
453 os.close(fd) |
|
454 if exists(newconfig): |
|
455 os.unlink(newconfig) |
|
456 |
|
457 # these are overridden by set_log_methods below |
|
458 # only defining here to prevent pylint from complaining |
|
459 info = warning = error = critical = exception = debug = lambda msg,*a,**kw: None |
|
460 |
|
461 from logging import getLogger |
|
462 from cubicweb import set_log_methods |
|
463 set_log_methods(MigrationHelper, getLogger('cubicweb.migration')) |
|
464 |
|
465 |
|
466 def version_strictly_lower(a, b): |
|
467 if a is None: |
|
468 return True |
|
469 if b is None: |
|
470 return False |
|
471 if a: |
|
472 a = Version(a) |
|
473 if b: |
|
474 b = Version(b) |
|
475 return a < b |
|
476 |
|
477 def max_version(a, b): |
|
478 return str(max(Version(a), Version(b))) |
|
479 |
|
480 class ConfigurationProblem(object): |
|
481 """Each cube has its own list of dependencies on other cubes/versions. |
|
482 |
|
483 The ConfigurationProblem is used to record the loaded cubes, then to detect |
|
484 inconsistencies in their dependencies. |
|
485 |
|
486 See configuration management on wikipedia for litterature. |
|
487 """ |
|
488 |
|
489 def __init__(self, config): |
|
490 self.config = config |
|
491 self.cubes = {'cubicweb': cwcfg.cubicweb_version()} |
|
492 |
|
493 def add_cube(self, name, version): |
|
494 self.cubes[name] = version |
|
495 |
|
496 def solve(self): |
|
497 self.warnings = [] |
|
498 self.errors = [] |
|
499 self.dependencies = {} |
|
500 self.reverse_dependencies = {} |
|
501 self.constraints = {} |
|
502 # read dependencies |
|
503 for cube in self.cubes: |
|
504 if cube == 'cubicweb': continue |
|
505 self.dependencies[cube] = dict(self.config.cube_dependencies(cube)) |
|
506 self.dependencies[cube]['cubicweb'] = self.config.cube_depends_cubicweb_version(cube) |
|
507 # compute reverse dependencies |
|
508 for cube, dependencies in self.dependencies.items(): |
|
509 for name, constraint in dependencies.items(): |
|
510 self.reverse_dependencies.setdefault(name,set()) |
|
511 if constraint: |
|
512 try: |
|
513 oper, version = constraint.split() |
|
514 self.reverse_dependencies[name].add( (oper, version, cube) ) |
|
515 except Exception: |
|
516 self.warnings.append( |
|
517 'cube %s depends on %s but constraint badly ' |
|
518 'formatted: %s' % (cube, name, constraint)) |
|
519 else: |
|
520 self.reverse_dependencies[name].add( (None, None, cube) ) |
|
521 # check consistency |
|
522 for cube, versions in sorted(self.reverse_dependencies.items()): |
|
523 oper, version, source = None, None, None |
|
524 # simplify constraints |
|
525 if versions: |
|
526 for constraint in versions: |
|
527 op, ver, src = constraint |
|
528 if oper is None: |
|
529 oper = op |
|
530 version = ver |
|
531 source = src |
|
532 elif op == '>=' and oper == '>=': |
|
533 if version_strictly_lower(version, ver): |
|
534 version = ver |
|
535 source = src |
|
536 elif op == None: |
|
537 continue |
|
538 else: |
|
539 print('unable to handle %s in %s, set to `%s %s` ' |
|
540 'but currently up to `%s %s`' % |
|
541 (cube, source, oper, version, op, ver)) |
|
542 # "solve" constraint satisfaction problem |
|
543 if cube not in self.cubes: |
|
544 self.errors.append( ('add', cube, version, source) ) |
|
545 elif versions: |
|
546 lower_strict = version_strictly_lower(self.cubes[cube], version) |
|
547 if oper in ('>=','=','=='): |
|
548 if lower_strict: |
|
549 self.errors.append( ('update', cube, version, source) ) |
|
550 elif oper is None: |
|
551 pass # no constraint on version |
|
552 else: |
|
553 print('unknown operator', oper) |