# HG changeset patch # User Sylvain Thénault # Date 1250003268 -7200 # Node ID 33f71848f7a7100aa453ed44fe8abbab008d2fb0 # Parent 3111abdad3b2fd61833498cf39aa4da7e6fa3dc3# Parent 58c519e5a31fbdc436f4856c67430819c3bbead5 backport some changesets to the stable branch diff -r 58c519e5a31f -r 33f71848f7a7 __pkginfo__.py --- a/__pkginfo__.py Tue Aug 11 17:04:59 2009 +0200 +++ b/__pkginfo__.py Tue Aug 11 17:07:48 2009 +0200 @@ -7,7 +7,7 @@ distname = "cubicweb" modname = "cubicweb" -numversion = (3, 4, 2) +numversion = (3, 4, 3) version = '.'.join(str(num) for num in numversion) license = 'LGPL v2' diff -r 58c519e5a31f -r 33f71848f7a7 server/migractions.py --- a/server/migractions.py Tue Aug 11 17:04:59 2009 +0200 +++ b/server/migractions.py Tue Aug 11 17:07:48 2009 +0200 @@ -19,7 +19,10 @@ import sys import os -from os.path import join, exists +import tarfile +import tempfile +import shutil +import os.path as osp from datetime import datetime from logilab.common.deprecation import deprecated @@ -110,25 +113,77 @@ def backup_database(self, backupfile=None, askconfirm=True): config = self.config repo = self.repo_connect() + # paths timestamp = datetime.now().strftime('%Y-%m-%d_%H:%M:%S') + instbkdir = osp.join(config.appdatahome, 'backup') + if not osp.exists(instbkdir): + os.makedirs(instbkdir) + backupfile = backupfile or osp.join(instbkdir, '%s-%s.tar.gz' + % (config.appid, timestamp)) + # check backup has to be done + if osp.exists(backupfile) and not \ + self.confirm('Backup file %s exists, overwrite it?' % backupfile): + print '-> no backup done.' + return + elif askconfirm and not self.confirm('Backup %s database?' % config.appid): + print '-> no backup done.' + return + open(backupfile,'w').close() # kinda lock + os.chmod(backupfile, 0600) + # backup + tmpdir = tempfile.mkdtemp(dir=instbkdir) for source in repo.sources: - source.backup(self.confirm, backupfile, timestamp, - askconfirm=askconfirm) + try: + source.backup(osp.join(tmpdir,source.uri)) + except Exception, exc: + print '-> error trying to backup [%s]' % exc + if not self.confirm('Continue anyway?', default='n'): + raise SystemExit(1) + bkup = tarfile.open(backupfile, 'w|gz') + for filename in os.listdir(tmpdir): + bkup.add(osp.join(tmpdir,filename), filename) + bkup.close() + shutil.rmtree(tmpdir) + # call hooks repo.hm.call_hooks('server_backup', repo=repo, timestamp=timestamp) + # done + print '-> backup file', backupfile def restore_database(self, backupfile, drop=True, systemonly=True, askconfirm=True): config = self.config repo = self.repo_connect() + # check + if not osp.exists(backupfile): + raise Exception("Backup file %s doesn't exist" % backupfile) + return + if askconfirm and not self.confirm('Restore %s database from %s ?' + % (config.appid, backupfile)): + return + # unpack backup + bkup = tarfile.open(backupfile, 'r|gz') + for name in bkup.getnames(): + if name[0] in '/.': + raise Exception('Security check failed, path starts with "/" or "."') + bkup.close() # XXX seek error if not close+open !?! + bkup = tarfile.open(backupfile, 'r|gz') + tmpdir = tempfile.mkdtemp() + bkup.extractall(path=tmpdir) if systemonly: - repo.system_source.restore(self.confirm, backupfile=backupfile, - drop=drop, askconfirm=askconfirm) + repo.system_source.restore(osp.join(tmpdir,'system'), drop=drop) else: - # in that case, backup file is expected to be a time stamp for source in repo.sources: - source.backup(self.confirm, timestamp=backupfile, drop=drop, - askconfirm=askconfirm) - repo.hm.call_hooks('server_restore', repo=repo, timestamp=backupfile) + try: + source.restore(osp.join(tmpdir, source.uri), drop=drop) + except Exception, exc: + print '-> error trying to restore [%s]' % exc + if not self.confirm('Continue anyway?', default='n'): + raise SystemExit(1) + bkup.close() + shutil.rmtree(tmpdir) + # call hooks + repo.hm.call_hooks('server_restore', repo=repo, timestamp=backupfile) + print '-> database restored.' @property def cnx(self): @@ -213,10 +268,10 @@ def exec_event_script(self, event, cubepath=None, funcname=None, *args, **kwargs): if cubepath: - apc = join(cubepath, 'migration', '%s.py' % event) + apc = osp.join(cubepath, 'migration', '%s.py' % event) else: - apc = join(self.config.migration_scripts_dir(), '%s.py' % event) - if exists(apc): + apc = osp.join(self.config.migration_scripts_dir(), '%s.py' % event) + if osp.exists(apc): if self.config.free_wheel: from cubicweb.server.hooks import setowner_after_add_entity self.repo.hm.unregister_hook(setowner_after_add_entity, diff -r 58c519e5a31f -r 33f71848f7a7 server/sources/__init__.py --- a/server/sources/__init__.py Tue Aug 11 17:04:59 2009 +0200 +++ b/server/sources/__init__.py Tue Aug 11 17:07:48 2009 +0200 @@ -95,30 +95,11 @@ """method called by the repository once ready to handle request""" pass - def backup_file(self, backupfile=None, timestamp=None): - """return a unique file name for a source's dump - - either backupfile or timestamp (used to generated a backup file name if - needed) should be specified. - """ - if backupfile is None: - config = self.repo.config - return join(config.appdatahome, 'backup', - '%s-%s-%s.dump' % (config.appid, timestamp, self.uri)) - # backup file is the system database backup file, add uri to it if not - # already there - base, ext = splitext(backupfile) - if not base.endswith('-%s' % self.uri): - return '%s-%s%s' % (base, self.uri, ext) - return backupfile - - def backup(self, confirm, backupfile=None, timestamp=None, - askconfirm=False): + def backup(self, backupfile): """method called to create a backup of source's data""" pass - def restore(self, confirm, backupfile=None, timestamp=None, drop=True, - askconfirm=False): + def restore(self, backupfile): """method called to restore a backup of source's data""" pass diff -r 58c519e5a31f -r 33f71848f7a7 server/sources/extlite.py --- a/server/sources/extlite.py Tue Aug 11 17:04:59 2009 +0200 +++ b/server/sources/extlite.py Tue Aug 11 17:07:48 2009 +0200 @@ -11,8 +11,7 @@ from os.path import join, exists from cubicweb import server -from cubicweb.server.sqlutils import (SQL_PREFIX, SQLAdapterMixIn, sqlexec, - sql_source_backup, sql_source_restore) +from cubicweb.server.sqlutils import SQL_PREFIX, SQLAdapterMixIn, sqlexec from cubicweb.server.sources import native, rql2sql from cubicweb.server.sources import AbstractSource, dbg_st_search, dbg_results @@ -94,18 +93,21 @@ AbstractSource.__init__(self, repo, appschema, source_config, *args, **kwargs) - def backup(self, confirm, backupfile=None, timestamp=None, askconfirm=False): - """method called to create a backup of source's data""" - backupfile = self.backup_file(backupfile, timestamp) - sql_source_backup(self, self.sqladapter, confirm, backupfile, - askconfirm) + def backup(self, backupfile): + """method called to create a backup of the source's data""" + self.close_pool_connections() + try: + self.sqladapter.backup_to_file(backupfile) + finally: + self.open_pool_connections() - def restore(self, confirm, backupfile=None, timestamp=None, drop=True, - askconfirm=False): + def restore(self, backupfile, drop): """method called to restore a backup of source's data""" - backupfile = self.backup_file(backupfile, timestamp) - sql_source_restore(self, self.sqladapter, confirm, backupfile, drop, - askconfirm) + self.close_pool_connections() + try: + self.sqladapter.restore_from_file(backupfile, drop) + finally: + self.open_pool_connections() @property def _sqlcnx(self): diff -r 58c519e5a31f -r 33f71848f7a7 server/sources/native.py --- a/server/sources/native.py Tue Aug 11 17:04:59 2009 +0200 +++ b/server/sources/native.py Tue Aug 11 17:07:48 2009 +0200 @@ -25,8 +25,7 @@ from cubicweb import UnknownEid, AuthenticationError, Binary, server from cubicweb.server.utils import crypt_password -from cubicweb.server.sqlutils import (SQL_PREFIX, SQLAdapterMixIn, - sql_source_backup, sql_source_restore) +from cubicweb.server.sqlutils import SQL_PREFIX, SQLAdapterMixIn from cubicweb.server.rqlannotation import set_qdata from cubicweb.server.sources import AbstractSource, dbg_st_search, dbg_results from cubicweb.server.sources.rql2sql import SQLGenerator @@ -207,17 +206,21 @@ pool.pool_reset() self.repo._free_pool(pool) - def backup(self, confirm, backupfile=None, timestamp=None, - askconfirm=False): - """method called to create a backup of source's data""" - backupfile = self.backup_file(backupfile, timestamp) - sql_source_backup(self, self, confirm, backupfile, askconfirm) + def backup(self, backupfile): + """method called to create a backup of the source's data""" + self.close_pool_connections() + try: + self.backup_to_file(backupfile) + finally: + self.open_pool_connections() - def restore(self, confirm, backupfile=None, timestamp=None, drop=True, - askconfirm=False): + def restore(self, backupfile, drop): """method called to restore a backup of source's data""" - backupfile = self.backup_file(backupfile, timestamp) - sql_source_restore(self, self, confirm, backupfile, drop, askconfirm) + self.close_pool_connections() + try: + self.restore_from_file(backupfile, drop) + finally: + self.open_pool_connections() def init(self): self.init_creating() diff -r 58c519e5a31f -r 33f71848f7a7 server/sqlutils.py --- a/server/sqlutils.py Tue Aug 11 17:04:59 2009 +0200 +++ b/server/sqlutils.py Tue Aug 11 17:07:48 2009 +0200 @@ -120,39 +120,6 @@ skip_relations=skip_relations)) return '\n'.join(output) - -def sql_source_backup(source, sqladapter, confirm, backupfile, - askconfirm=False): - if exists(backupfile): - if not confirm('Backup file %s exists, overwrite it?' % backupfile): - return - elif askconfirm and not confirm('Backup %s database?' - % source.repo.config.appid): - print '-> no backup done.' - return - # should close opened connection before backuping - source.close_pool_connections() - try: - sqladapter.backup_to_file(backupfile, confirm) - finally: - source.open_pool_connections() - -def sql_source_restore(source, sqladapter, confirm, backupfile, drop=True, - askconfirm=False): - if not exists(backupfile): - raise Exception("backup file %s doesn't exist" % backupfile) - app = source.repo.config.appid - if askconfirm and not confirm('Restore %s %s database from %s ?' - % (app, source.uri, backupfile)): - return - # should close opened connection before restoring - source.close_pool_connections() - try: - sqladapter.restore_from_file(backupfile, confirm, drop=drop) - finally: - source.open_pool_connections() - - try: from mx.DateTime import DateTimeType, DateTimeDeltaType except ImportError: @@ -196,25 +163,12 @@ #self.dbapi_module.type_code_test(cnx.cursor()) return cnx - def backup_to_file(self, backupfile, confirm): + def backup_to_file(self, backupfile): cmd = self.dbhelper.backup_command(self.dbname, self.dbhost, self.dbuser, backupfile, keepownership=False) - backupdir = os.path.dirname(backupfile) - if not os.path.exists(backupdir): - if confirm('%s does not exist. Create it?' % backupdir, - abort=False, shell=False): - os.makedirs(backupdir) - else: - print '-> failed to backup instance' - return if os.system(cmd): - print '-> error trying to backup with command', cmd - if not confirm('Continue anyway?', default='n'): - raise SystemExit(1) - else: - print '-> backup file', backupfile - restrict_perms_to_user(backupfile, self.info) + raise Exception('Failed command: %s' % cmd) def restore_from_file(self, backupfile, confirm, drop=True): for cmd in self.dbhelper.restore_commands(self.dbname, self.dbhost, @@ -222,19 +176,8 @@ self.encoding, keepownership=False, drop=drop): - while True: - print cmd - if os.system(cmd): - print '-> error while restoring the base' - answer = confirm('Continue anyway?', - shell=False, abort=False, retry=True) - if not answer: - raise SystemExit(1) - if answer == 1: # 1: continue, 2: retry - break - else: - break - print '-> database restored.' + if os.system(cmd): + raise Exception('Failed command: %s' % cmd) def merge_args(self, args, query_args): if args is not None: