|
1 """utilities for instances migration |
|
2 |
|
3 :organization: Logilab |
|
4 :copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2. |
|
5 :contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr |
|
6 :license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses |
|
7 """ |
|
8 __docformat__ = "restructuredtext en" |
|
9 |
|
10 import sys |
|
11 import os |
|
12 import logging |
|
13 import tempfile |
|
14 from os.path import exists, join, basename, splitext |
|
15 |
|
16 from logilab.common.decorators import cached |
|
17 from logilab.common.configuration import REQUIRED, read_old_config |
|
18 from logilab.common.shellutils import ASK |
|
19 |
|
20 from cubicweb import ConfigurationError |
|
21 |
|
22 |
|
23 def filter_scripts(config, directory, fromversion, toversion, quiet=True): |
|
24 """return a list of paths of migration files to consider to upgrade |
|
25 from a version to a greater one |
|
26 """ |
|
27 from logilab.common.changelog import Version # doesn't work with appengine |
|
28 assert fromversion |
|
29 assert toversion |
|
30 assert isinstance(fromversion, tuple), fromversion.__class__ |
|
31 assert isinstance(toversion, tuple), toversion.__class__ |
|
32 assert fromversion <= toversion, (fromversion, toversion) |
|
33 if not exists(directory): |
|
34 if not quiet: |
|
35 print directory, "doesn't exists, no migration path" |
|
36 return [] |
|
37 if fromversion == toversion: |
|
38 return [] |
|
39 result = [] |
|
40 for fname in os.listdir(directory): |
|
41 if fname.endswith('.pyc') or fname.endswith('.pyo') \ |
|
42 or fname.endswith('~'): |
|
43 continue |
|
44 fpath = join(directory, fname) |
|
45 try: |
|
46 tver, mode = fname.split('_', 1) |
|
47 except ValueError: |
|
48 continue |
|
49 mode = mode.split('.', 1)[0] |
|
50 if not config.accept_mode(mode): |
|
51 continue |
|
52 try: |
|
53 tver = Version(tver) |
|
54 except ValueError: |
|
55 continue |
|
56 if tver <= fromversion: |
|
57 continue |
|
58 if tver > toversion: |
|
59 continue |
|
60 result.append((tver, fpath)) |
|
61 # be sure scripts are executed in order |
|
62 return sorted(result) |
|
63 |
|
64 |
|
65 IGNORED_EXTENSIONS = ('.swp', '~') |
|
66 |
|
67 |
|
68 def execscript_confirm(scriptpath): |
|
69 """asks for confirmation before executing a script and provides the |
|
70 ability to show the script's content |
|
71 """ |
|
72 while True: |
|
73 answer = ASK.ask('Execute %r ?' % scriptpath, |
|
74 ('Y','n','show','abort'), 'Y') |
|
75 if answer == 'abort': |
|
76 raise SystemExit(1) |
|
77 elif answer == 'n': |
|
78 return False |
|
79 elif answer == 'show': |
|
80 stream = open(scriptpath) |
|
81 scriptcontent = stream.read() |
|
82 stream.close() |
|
83 print |
|
84 print scriptcontent |
|
85 print |
|
86 else: |
|
87 return True |
|
88 |
|
89 def yes(*args, **kwargs): |
|
90 return True |
|
91 |
|
92 |
|
93 class MigrationHelper(object): |
|
94 """class holding CubicWeb Migration Actions used by migration scripts""" |
|
95 |
|
96 def __init__(self, config, interactive=True, verbosity=1): |
|
97 self.config = config |
|
98 if config: |
|
99 # no config on shell to a remote instance |
|
100 self.config.init_log(logthreshold=logging.ERROR, debug=True) |
|
101 # 0: no confirmation, 1: only main commands confirmed, 2 ask for everything |
|
102 self.verbosity = verbosity |
|
103 self.need_wrap = True |
|
104 if not interactive or not verbosity: |
|
105 self.confirm = yes |
|
106 self.execscript_confirm = yes |
|
107 else: |
|
108 self.execscript_confirm = execscript_confirm |
|
109 self._option_changes = [] |
|
110 self.__context = {'confirm': self.confirm, |
|
111 'config': self.config, |
|
112 'interactive_mode': interactive, |
|
113 } |
|
114 |
|
115 def __getattribute__(self, name): |
|
116 try: |
|
117 return object.__getattribute__(self, name) |
|
118 except AttributeError: |
|
119 cmd = 'cmd_%s' % name |
|
120 if hasattr(self, cmd): |
|
121 meth = getattr(self, cmd) |
|
122 return lambda *args, **kwargs: self.interact(args, kwargs, |
|
123 meth=meth) |
|
124 raise |
|
125 raise AttributeError(name) |
|
126 |
|
127 def repo_connect(self): |
|
128 return self.config.repository() |
|
129 |
|
130 def migrate(self, vcconf, toupgrade, options): |
|
131 """upgrade the given set of cubes |
|
132 |
|
133 `cubes` is an ordered list of 3-uple: |
|
134 (cube, fromversion, toversion) |
|
135 """ |
|
136 if options.fs_only: |
|
137 # monkey path configuration.accept_mode so database mode (e.g. Any) |
|
138 # won't be accepted |
|
139 orig_accept_mode = self.config.accept_mode |
|
140 def accept_mode(mode): |
|
141 if mode == 'Any': |
|
142 return False |
|
143 return orig_accept_mode(mode) |
|
144 self.config.accept_mode = accept_mode |
|
145 # may be an iterator |
|
146 toupgrade = tuple(toupgrade) |
|
147 vmap = dict( (cube, (fromver, tover)) for cube, fromver, tover in toupgrade) |
|
148 ctx = self.__context |
|
149 ctx['versions_map'] = vmap |
|
150 if self.config.accept_mode('Any') and 'cubicweb' in vmap: |
|
151 migrdir = self.config.migration_scripts_dir() |
|
152 self.cmd_process_script(join(migrdir, 'bootstrapmigration_repository.py')) |
|
153 for cube, fromversion, toversion in toupgrade: |
|
154 if cube == 'cubicweb': |
|
155 migrdir = self.config.migration_scripts_dir() |
|
156 else: |
|
157 migrdir = self.config.cube_migration_scripts_dir(cube) |
|
158 scripts = filter_scripts(self.config, migrdir, fromversion, toversion) |
|
159 if scripts: |
|
160 prevversion = None |
|
161 for version, script in scripts: |
|
162 # take care to X.Y.Z_Any.py / X.Y.Z_common.py: we've to call |
|
163 # cube_upgraded once all script of X.Y.Z have been executed |
|
164 if prevversion is not None and version != prevversion: |
|
165 self.cube_upgraded(cube, prevversion) |
|
166 prevversion = version |
|
167 self.cmd_process_script(script) |
|
168 self.cube_upgraded(cube, toversion) |
|
169 else: |
|
170 self.cube_upgraded(cube, toversion) |
|
171 |
|
172 def cube_upgraded(self, cube, version): |
|
173 pass |
|
174 |
|
175 def shutdown(self): |
|
176 pass |
|
177 |
|
178 def interact(self, args, kwargs, meth): |
|
179 """execute the given method according to user's confirmation""" |
|
180 msg = 'Execute command: %s(%s) ?' % ( |
|
181 meth.__name__[4:], |
|
182 ', '.join([repr(arg) for arg in args] + |
|
183 ['%s=%r' % (n,v) for n,v in kwargs.items()])) |
|
184 if 'ask_confirm' in kwargs: |
|
185 ask_confirm = kwargs.pop('ask_confirm') |
|
186 else: |
|
187 ask_confirm = True |
|
188 if not ask_confirm or self.confirm(msg): |
|
189 return meth(*args, **kwargs) |
|
190 |
|
191 def confirm(self, question, shell=True, abort=True, retry=False, default='y'): |
|
192 """ask for confirmation and return true on positive answer |
|
193 |
|
194 if `retry` is true the r[etry] answer may return 2 |
|
195 """ |
|
196 possibleanswers = ['y', 'n'] |
|
197 if abort: |
|
198 possibleanswers.append('abort') |
|
199 if shell: |
|
200 possibleanswers.append('shell') |
|
201 if retry: |
|
202 possibleanswers.append('retry') |
|
203 try: |
|
204 answer = ASK.ask(question, possibleanswers, default) |
|
205 except (EOFError, KeyboardInterrupt): |
|
206 answer = 'abort' |
|
207 if answer == 'n': |
|
208 return False |
|
209 if answer == 'retry': |
|
210 return 2 |
|
211 if answer == 'abort': |
|
212 raise SystemExit(1) |
|
213 if shell and answer == 'shell': |
|
214 self.interactive_shell() |
|
215 return self.confirm(question) |
|
216 return True |
|
217 |
|
218 def interactive_shell(self): |
|
219 self.confirm = yes |
|
220 self.need_wrap = False |
|
221 # avoid '_' to be added to builtins by sys.display_hook |
|
222 def do_not_add___to_builtins(obj): |
|
223 if obj is not None: |
|
224 print repr(obj) |
|
225 sys.displayhook = do_not_add___to_builtins |
|
226 local_ctx = self._create_context() |
|
227 try: |
|
228 import readline |
|
229 from rlcompleter import Completer |
|
230 except ImportError: |
|
231 # readline not available |
|
232 pass |
|
233 else: |
|
234 readline.set_completer(Completer(local_ctx).complete) |
|
235 readline.parse_and_bind('tab: complete') |
|
236 home_key = 'HOME' |
|
237 if sys.platform == 'win32': |
|
238 home_key = 'USERPROFILE' |
|
239 histfile = os.path.join(os.environ[home_key], ".eshellhist") |
|
240 try: |
|
241 readline.read_history_file(histfile) |
|
242 except IOError: |
|
243 pass |
|
244 from code import interact |
|
245 banner = """entering the migration python shell |
|
246 just type migration commands or arbitrary python code and type ENTER to execute it |
|
247 type "exit" or Ctrl-D to quit the shell and resume operation""" |
|
248 # give custom readfunc to avoid http://bugs.python.org/issue1288615 |
|
249 def unicode_raw_input(prompt): |
|
250 return unicode(raw_input(prompt), sys.stdin.encoding) |
|
251 interact(banner, readfunc=unicode_raw_input, local=local_ctx) |
|
252 readline.write_history_file(histfile) |
|
253 # delete instance's confirm attribute to avoid questions |
|
254 del self.confirm |
|
255 self.need_wrap = True |
|
256 |
|
257 @cached |
|
258 def _create_context(self): |
|
259 """return a dictionary to use as migration script execution context""" |
|
260 context = self.__context |
|
261 for attr in dir(self): |
|
262 if attr.startswith('cmd_'): |
|
263 if self.need_wrap: |
|
264 context[attr[4:]] = getattr(self, attr[4:]) |
|
265 else: |
|
266 context[attr[4:]] = getattr(self, attr) |
|
267 return context |
|
268 |
|
269 def cmd_process_script(self, migrscript, funcname=None, *args, **kwargs): |
|
270 """execute a migration script |
|
271 in interactive mode, display the migration script path, ask for |
|
272 confirmation and execute it if confirmed |
|
273 """ |
|
274 migrscript = os.path.normpath(migrscript) |
|
275 if migrscript.endswith('.py'): |
|
276 script_mode = 'python' |
|
277 elif migrscript.endswith('.txt') or migrscript.endswith('.rst'): |
|
278 script_mode = 'doctest' |
|
279 else: |
|
280 raise Exception('This is not a valid cubicweb shell input') |
|
281 if not self.execscript_confirm(migrscript): |
|
282 return |
|
283 scriptlocals = self._create_context().copy() |
|
284 if script_mode == 'python': |
|
285 if funcname is None: |
|
286 pyname = '__main__' |
|
287 else: |
|
288 pyname = splitext(basename(migrscript))[0] |
|
289 scriptlocals.update({'__file__': migrscript, '__name__': pyname}) |
|
290 execfile(migrscript, scriptlocals) |
|
291 if funcname is not None: |
|
292 try: |
|
293 func = scriptlocals[funcname] |
|
294 self.info('found %s in locals', funcname) |
|
295 assert callable(func), '%s (%s) is not callable' % (func, funcname) |
|
296 except KeyError: |
|
297 self.critical('no %s in script %s', funcname, migrscript) |
|
298 return None |
|
299 return func(*args, **kwargs) |
|
300 else: # script_mode == 'doctest' |
|
301 import doctest |
|
302 doctest.testfile(migrscript, module_relative=False, |
|
303 optionflags=doctest.ELLIPSIS, globs=scriptlocals) |
|
304 |
|
305 def cmd_option_renamed(self, oldname, newname): |
|
306 """a configuration option has been renamed""" |
|
307 self._option_changes.append(('renamed', oldname, newname)) |
|
308 |
|
309 def cmd_option_group_change(self, option, oldgroup, newgroup): |
|
310 """a configuration option has been moved in another group""" |
|
311 self._option_changes.append(('moved', option, oldgroup, newgroup)) |
|
312 |
|
313 def cmd_option_added(self, optname): |
|
314 """a configuration option has been added""" |
|
315 self._option_changes.append(('added', optname)) |
|
316 |
|
317 def cmd_option_removed(self, optname): |
|
318 """a configuration option has been removed""" |
|
319 # can safely be ignored |
|
320 #self._option_changes.append(('removed', optname)) |
|
321 |
|
322 def cmd_option_type_changed(self, optname, oldtype, newvalue): |
|
323 """a configuration option's type has changed""" |
|
324 self._option_changes.append(('typechanged', optname, oldtype, newvalue)) |
|
325 |
|
326 def cmd_add_cubes(self, cubes): |
|
327 """modify the list of used cubes in the in-memory config |
|
328 returns newly inserted cubes, including dependencies |
|
329 """ |
|
330 if isinstance(cubes, basestring): |
|
331 cubes = (cubes,) |
|
332 origcubes = self.config.cubes() |
|
333 newcubes = [p for p in self.config.expand_cubes(cubes) |
|
334 if not p in origcubes] |
|
335 if newcubes: |
|
336 for cube in cubes: |
|
337 assert cube in newcubes |
|
338 self.config.add_cubes(newcubes) |
|
339 return newcubes |
|
340 |
|
341 def cmd_remove_cube(self, cube, removedeps=False): |
|
342 if removedeps: |
|
343 toremove = self.config.expand_cubes([cube]) |
|
344 else: |
|
345 toremove = (cube,) |
|
346 origcubes = self.config._cubes |
|
347 basecubes = [c for c in origcubes if not c in toremove] |
|
348 self.config._cubes = tuple(self.config.expand_cubes(basecubes)) |
|
349 removed = [p for p in origcubes if not p in self.config._cubes] |
|
350 if not cube in removed: |
|
351 raise ConfigurationError("can't remove cube %s, " |
|
352 "used as a dependency" % cube) |
|
353 return removed |
|
354 |
|
355 def rewrite_configuration(self): |
|
356 # import locally, show_diffs unavailable in gae environment |
|
357 from cubicweb.toolsutils import show_diffs |
|
358 configfile = self.config.main_config_file() |
|
359 if self._option_changes: |
|
360 read_old_config(self.config, self._option_changes, configfile) |
|
361 fd, newconfig = tempfile.mkstemp() |
|
362 for optdescr in self._option_changes: |
|
363 if optdescr[0] == 'added': |
|
364 optdict = self.config.get_option_def(optdescr[1]) |
|
365 if optdict.get('default') is REQUIRED: |
|
366 self.config.input_option(optdescr[1], optdict) |
|
367 self.config.generate_config(open(newconfig, 'w')) |
|
368 show_diffs(configfile, newconfig) |
|
369 os.close(fd) |
|
370 if exists(newconfig): |
|
371 os.unlink(newconfig) |
|
372 |
|
373 |
|
374 from logging import getLogger |
|
375 from cubicweb import set_log_methods |
|
376 set_log_methods(MigrationHelper, getLogger('cubicweb.migration')) |