diff -r 000000000000 -r b97547f5f1fa server/serverctl.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/server/serverctl.py Wed Nov 05 15:52:50 2008 +0100 @@ -0,0 +1,733 @@ +"""cubicweb-ctl commands and command handlers specific to the server.serverconfig + +:organization: Logilab +:copyright: 2001-2008 LOGILAB S.A. (Paris, FRANCE), all rights reserved. +:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr +""" +__docformat__ = "restructuredtext en" + +import os + +from logilab.common.configuration import REQUIRED, Configuration, ini_format_section + +from cubicweb import AuthenticationError, ExecutionError, ConfigurationError +from cubicweb.toolsutils import Command, CommandHandler, pop_arg, cmd_run, \ + register_commands, confirm, restrict_perms_to_user +from cubicweb.server.serverconfig import ServerConfiguration + + +# utility functions ########################################################### + +def source_cnx(source, dbname=None, special_privs=False): + """open and return a connection to the system database defined in the + given server.serverconfig + """ + from getpass import getpass + from logilab.common.db import get_connection + dbhost = source['db-host'] + if dbname is None: + dbname = source['db-name'] + driver = source['db-driver'] + print '**** connecting to %s database %s@%s' % (driver, dbname, dbhost), + if not special_privs and source.get('db-user'): + user = source['db-user'] + print 'as', user + if source.get('db-password'): + password = source['db-password'] + else: + password = getpass('password: ') + else: + print + if special_privs: + print 'WARNING' + print 'the user will need the following special access rights on the database:' + print special_privs + print + default_user = source.get('db-user', os.environ.get('USER', '')) + user = raw_input('user (%r by default): ' % default_user) + user = user or default_user + if user == source.get('db-user') and source.get('db-password'): + password = source['db-password'] + else: + password = getpass('password: ') + return get_connection(driver, dbhost, dbname, user, password=password, + port=source.get('db-port')) + +def system_source_cnx(source, dbms_system_base=False, special_privs=None): + """shortcut to get a connextion to the application system database + defined in the given config. If is True, + connect to the dbms system database instead (for task such as + create/drop the application database) + """ + if dbms_system_base: + from logilab.common.adbh import get_adv_func_helper + system_db = get_adv_func_helper(source['db-driver']).system_database() + special_privs = special_privs or 'CREATE/DROP DATABASE' + return source_cnx(source, system_db, special_privs=special_privs) + return source_cnx(source, special_privs=special_privs) + +def _db_sys_cnx(source, what, db=None, user=None): + """return a connection on the RDMS system table (to create/drop a user + or a database + """ + from logilab.common.adbh import get_adv_func_helper + special_privs = '' + driver = source['db-driver'] + helper = get_adv_func_helper(driver) + if user is not None and helper.users_support: + special_privs += '%s USER' % what + if db is not None: + special_privs += ' %s DATABASE' % what + # connect on the dbms system base to create our base + cnx = system_source_cnx(source, True, special_privs=special_privs) + # disable autocommit (isolation_level(1)) because DROP and + # CREATE DATABASE can't be executed in a transaction + try: + cnx.set_isolation_level(0) + except AttributeError: + # set_isolation_level() is psycopg specific + pass + return cnx + +def generate_sources_file(sourcesfile, sourcescfg, keys=None): + """serialize repository'sources configuration into a INI like file + + the `keys` parameter may be used to sort sections + """ + from cubicweb.server.sources import SOURCE_TYPES + if keys is None: + keys = sourcescfg.keys() + else: + for key in sourcescfg: + if not key in keys: + keys.append(key) + stream = open(sourcesfile, 'w') + for uri in keys: + sconfig = sourcescfg[uri] + if isinstance(sconfig, dict): + # get a Configuration object + _sconfig = Configuration(options=SOURCE_TYPES[sconfig['adapter']].options) + for attr, val in sconfig.items(): + if attr == 'uri': + continue + if attr == 'adapter': + _sconfig.adapter = val + else: + _sconfig.set_option(attr, val) + sconfig = _sconfig + optsbysect = list(sconfig.options_by_section()) + assert len(optsbysect) == 1 + ini_format_section(stream, uri, optsbysect[0][1]) + if hasattr(sconfig, 'adapter'): + print >> stream + print >> stream, '# adapter for this source (YOU SHOULD NOT CHANGE THIS)' + print >> stream, 'adapter=%s' % sconfig.adapter + print >> stream + +def repo_cnx(config): + """return a in-memory repository and a db api connection it""" + from cubicweb.dbapi import in_memory_cnx + from cubicweb.server.utils import manager_userpasswd + try: + login = config.sources()['admin']['login'] + pwd = config.sources()['admin']['password'] + except KeyError: + login, pwd = manager_userpasswd() + while True: + try: + return in_memory_cnx(config, login, pwd) + except AuthenticationError: + print 'wrong user/password' + login, pwd = manager_userpasswd() + +# repository specific command handlers ######################################## + +class RepositoryCreateHandler(CommandHandler): + cmdname = 'create' + cfgname = 'repository' + + def bootstrap(self, cubes, inputlevel=0): + """create an application by copying files from the given cube and by + asking information necessary to build required configuration files + """ + from cubicweb.server.sources import SOURCE_TYPES + config = self.config + print 'application\'s repository configuration' + print '-' * 72 + config.input_config('email', inputlevel) + if config.pyro_enabled(): + config.input_config('pyro-server', inputlevel) + print + print 'repository sources configuration' + print '-' * 72 + sourcesfile = config.sources_file() + sconfig = Configuration(options=SOURCE_TYPES['native'].options) + sconfig.adapter = 'native' + sconfig.input_config(inputlevel=inputlevel) + sourcescfg = {'system': sconfig} + while raw_input('enter another source [y/N]: ').strip().lower() == 'y': + sourcetype = raw_input('source type (%s): ' % ', '.join(SOURCE_TYPES.keys())) + sconfig = Configuration(options=SOURCE_TYPES[sourcetype].options) + sconfig.adapter = sourcetype + sourceuri = raw_input('source uri: ').strip() + assert not sourceuri in sourcescfg + sconfig.input_config(inputlevel=inputlevel) + sourcescfg[sourceuri] = sconfig + # module names look like cubes.mycube.themodule + sourcecube = SOURCE_TYPES[sourcetype].module.split('.', 2)[1] + # if the source adapter is coming from an external component, ensure + # it's specified in used cubes + if sourcecube != 'cubicweb' and not sourcecube in cubes: + cubes.append(sourcecube) + sconfig = Configuration(options=USER_OPTIONS) + sconfig.input_config(inputlevel=inputlevel) + sourcescfg['admin'] = sconfig + generate_sources_file(sourcesfile, sourcescfg, ['admin', 'system']) + restrict_perms_to_user(sourcesfile) + # remember selected cubes for later initialization of the database + config.write_bootstrap_cubes_file(cubes) + + def postcreate(self): + if confirm('do you want to create repository\'s system database?'): + cmd_run('db-create', self.config.appid) + else: + print 'nevermind, you can do it later using the db-create command' + +USER_OPTIONS = ( + ('login', {'type' : 'string', + 'default': REQUIRED, + 'help': "cubicweb manager account's login " + '(this user will be created)', + 'inputlevel': 0, + }), + ('password', {'type' : 'password', + 'help': "cubicweb manager account's password", + 'inputlevel': 0, + }), + ) + + +class RepositoryDeleteHandler(CommandHandler): + cmdname = 'delete' + cfgname = 'repository' + + def cleanup(self): + """remove application's configuration and database""" + from logilab.common.adbh import get_adv_func_helper + source = self.config.sources()['system'] + dbname = source['db-name'] + helper = get_adv_func_helper(source['db-driver']) + if confirm('delete database %s ?' % dbname): + user = source['db-user'] or None + cnx = _db_sys_cnx(source, 'DROP DATABASE', user=user) + cursor = cnx.cursor() + try: + cursor.execute('DROP DATABASE %s' % dbname) + print 'database %s dropped' % dbname + # XXX should check we are not connected as user + if user and helper.users_support and \ + confirm('delete user %s ?' % user, default_is_yes=False): + cursor.execute('DROP USER %s' % user) + print 'user %s dropped' % user + cnx.commit() + except: + cnx.rollback() + raise + + +class RepositoryStartHandler(CommandHandler): + cmdname = 'start' + cfgname = 'repository' + + def start_command(self, ctlconf, debug): + command = ['cubicweb-ctl start-repository '] + if debug: + command.append('--debug') + command.append(self.config.appid) + return ' '.join(command) + + +class RepositoryStopHandler(CommandHandler): + cmdname = 'stop' + cfgname = 'repository' + + def poststop(self): + """if pyro is enabled, ensure the repository is correctly + unregistered + """ + if self.config.pyro_enabled(): + from cubicweb.server.repository import pyro_unregister + pyro_unregister(self.config) + + +# repository specific commands ################################################ +class CreateApplicationDBCommand(Command): + """Create the system database of an application (run after 'create'). + + You will be prompted for a login / password to use to connect to + the system database. The given user should have almost all rights + on the database (ie a super user on the dbms allowed to create + database, users, languages...). + + + the identifier of the application to initialize. + """ + name = 'db-create' + arguments = '' + + options = ( + ("create-db", + {'short': 'c', 'type': "yn", 'metavar': '', + 'default': True, + 'help': 'create the database (yes by default)'}), + ) + def run(self, args): + """run the command with its specific arguments""" + from logilab.common.adbh import get_adv_func_helper + from indexer import get_indexer + appid = pop_arg(args, msg="No application specified !") + config = ServerConfiguration.config_for(appid) + create_db = self.config.create_db + source = config.sources()['system'] + driver = source['db-driver'] + helper = get_adv_func_helper(driver) + if create_db: + # connect on the dbms system base to create our base + dbcnx = _db_sys_cnx(source, 'CREATE DATABASE and / or USER') + cursor = dbcnx.cursor() + try: + if helper.users_support: + user = source['db-user'] + if not helper.user_exists(cursor, user) and \ + confirm('create db user %s ?' % user, default_is_yes=False): + helper.create_user(source['db-user'], source['db-password']) + print 'user %s created' % user + dbname = source['db-name'] + if dbname in helper.list_databases(cursor): + if confirm('DB %s already exists -- do you want to drop it ?' % dbname): + cursor.execute('DROP DATABASE %s' % dbname) + else: + return + if dbcnx.logged_user != source['db-user']: + helper.create_database(cursor, dbname, source['db-user'], + source['db-encoding']) + else: + helper.create_database(cursor, dbname, + encoding=source['db-encoding']) + dbcnx.commit() + print 'database %s created' % source['db-name'] + except: + dbcnx.rollback() + raise + cnx = system_source_cnx(source, special_privs='LANGUAGE C') + cursor = cnx.cursor() + indexer = get_indexer(driver) + indexer.init_extensions(cursor) + # postgres specific stuff + if driver == 'postgres': + # install plpythonu/plpgsql language if not installed by the cube + for extlang in ('plpythonu', 'plpgsql'): + helper.create_language(cursor, extlang) + cursor.close() + cnx.commit() + print 'database for application %s created and necessary extensions installed' % appid + print + if confirm('do you want to initialize the system database?'): + cmd_run('db-init', config.appid) + else: + print 'nevermind, you can do it later using the db-init command' + + +class InitApplicationCommand(Command): + """Initialize the system database of an application (run after 'db-create'). + + You will be prompted for a login / password to use to connect to + the system database. The given user should have the create tables, + and grant permissions. + + + the identifier of the application to initialize. + """ + name = 'db-init' + arguments = '' + + options = ( + ("drop", + {'short': 'd', 'action': 'store_true', + 'default': False, + 'help': 'insert drop statements to remove previously existant \ +tables, indexes... (no by default)'}), + ) + + def run(self, args): + from cubicweb.server import init_repository + appid = pop_arg(args, msg="No application specified !") + config = ServerConfiguration.config_for(appid) + init_repository(config, drop=self.config.drop) + + +class GrantUserOnApplicationCommand(Command): + """Grant a database user on a repository system database. + + + the identifier of the application + + the database's user requiring grant access + """ + name = 'db-grant-user' + arguments = ' ' + + options = ( + ("set-owner", + {'short': 'o', 'type' : "yn", 'metavar' : '', + 'default' : False, + 'help': 'Set the user as tables owner if yes (no by default).'} + ), + ) + def run(self, args): + """run the command with its specific arguments""" + from cubicweb.server.sqlutils import sqlexec, sqlgrants + appid = pop_arg(args, 1, msg="No application specified !") + user = pop_arg(args, msg="No user specified !") + config = ServerConfiguration.config_for(appid) + source = config.sources()['system'] + set_owner = self.config.set_owner + cnx = system_source_cnx(source, special_privs='GRANT') + cursor = cnx.cursor() + schema = config.load_schema() + try: + sqlexec(sqlgrants(schema, source['db-driver'], user, + set_owner=set_owner), cursor) + except Exception, ex: + cnx.rollback() + import traceback + traceback.print_exc() + print 'An error occured:', ex + else: + cnx.commit() + print 'grants given to %s on application %s' % (appid, user) + + + +class StartRepositoryCommand(Command): + """Start an CubicWeb RQL server for a given application. + + The server will be accessible through pyro + + + the identifier of the application to initialize. + """ + name = 'start-repository' + arguments = '' + + options = ( + ("debug", + {'short': 'D', 'action' : 'store_true', + 'help': 'start server in debug mode.'}), + ) + + def run(self, args): + from cubicweb.server.server import RepositoryServer + appid = pop_arg(args, msg="No application specified !") + config = ServerConfiguration.config_for(appid) + debug = self.config.debug + # create the server + server = RepositoryServer(config, debug) + # go ! (don't daemonize in debug mode) + if not debug and server.daemonize(config['pid-file']) == -1: + return + uid = config['uid'] + if uid is not None: + try: + uid = int(uid) + except ValueError: + from pwd import getpwnam + uid = getpwnam(uid).pw_uid + os.setuid(uid) + server.install_sig_handlers() + server.connect(config['host'], 0) + server.run() + + +def _remote_dump(host, appid, output, sudo=False): + dmpcmd = 'cubicweb-ctl db-dump -o /tmp/%s.dump %s' % (appid, appid) + if sudo: + dmpcmd = 'sudo %s' % (dmpcmd) + dmpcmd = 'ssh -t %s "%s"' % (host, dmpcmd) + print dmpcmd + if os.system(dmpcmd): + raise ExecutionError('Error while dumping the database') + if output is None: + from mx.DateTime import today + date = today().strftime('%Y-%m-%d') + output = '%s-%s.dump' % (appid, date) + cmd = 'scp %s:/tmp/%s.dump %s' % (host, appid, output) + print cmd + if os.system(cmd): + raise ExecutionError('Error while retrieving the dump') + rmcmd = 'ssh -t %s "rm -f /tmp/%s.dump"' % (host, appid) + print rmcmd + if os.system(rmcmd) and not confirm('an error occured while deleting remote dump. Continue anyway?'): + raise ExecutionError('Error while deleting remote dump') + +def _local_dump(appid, output): + config = ServerConfiguration.config_for(appid) + # schema=1 to avoid unnecessary schema loading + mih = config.migration_handler(connect=False, schema=1) + mih.backup_database(output, askconfirm=False) + +def _local_restore(appid, backupfile, drop): + config = ServerConfiguration.config_for(appid) + # schema=1 to avoid unnecessary schema loading + mih = config.migration_handler(connect=False, schema=1) + mih.restore_database(backupfile, drop) + repo = mih.repo_connect() + # version of the database + dbversions = repo.get_versions() + mih.shutdown() + if not dbversions: + print "bad or missing version information in the database, don't upgrade file system" + return + # version of installed software + eversion = dbversions['cubicweb'] + status = application_status(config, eversion, dbversions) + # * database version > installed software + if status == 'needsoftupgrade': + print "database is using some earlier version than installed software!" + print "please upgrade your software and then upgrade the instance" + print "using command 'cubicweb-ctl upgrade %s'" % config.appid + return + # * database version < installed software, an upgrade will be necessary + # anyway, just rewrite vc.conf and warn user he has to upgrade + if status == 'needapplupgrade': + print "database is using some older version than installed software." + print "You'll have to upgrade the instance using command" + print "'cubicweb-ctl upgrade %s'" % config.appid + return + # * database version = installed software, database version = instance fs version + # ok! + + +def application_status(config, cubicwebapplversion, vcconf): + cubicwebversion = config.cubicweb_version() + if cubicwebapplversion > cubicwebversion: + return 'needsoftupgrade' + if cubicwebapplversion < cubicwebversion: + return 'needapplupgrade' + for cube in config.cubes(): + try: + softversion = config.cube_version(cube) + except ConfigurationError: + print "no cube version information for %s, is the cube installed?" % cube + continue + try: + applversion = vcconf[cube] + except KeyError: + print "no cube version information for %s in version configuration" % cube + continue + if softversion == applversion: + continue + if softversion > applversion: + return 'needsoftupgrade' + elif softversion < applversion: + return 'needapplupgrade' + return None + + +class DBDumpCommand(Command): + """Backup the system database of an application. + + + the identifier of the application to backup + format [[user@]host:]appname + """ + name = 'db-dump' + arguments = '' + + options = ( + ("output", + {'short': 'o', 'type' : "string", 'metavar' : '', + 'default' : None, + 'help': 'Specify the backup file where the backup will be stored.'} + ), + ('sudo', + {'short': 's', 'action' : 'store_true', + 'default' : False, + 'help': 'Use sudo on the remote host.'} + ), + ) + + def run(self, args): + appid = pop_arg(args, 1, msg="No application specified !") + if ':' in appid: + host, appid = appid.split(':') + _remote_dump(host, appid, self.config.output, self.config.sudo) + else: + _local_dump(appid, self.config.output) + + +class DBRestoreCommand(Command): + """Restore the system database of an application. + + + the identifier of the application to restore + """ + name = 'db-restore' + arguments = ' ' + + options = ( + ("no-drop", + {'short': 'n', 'action' : 'store_true', + 'default' : False, + 'help': 'for some reason the database doesn\'t exist and so ' + 'should not be dropped.'} + ), + ) + + def run(self, args): + appid = pop_arg(args, 1, msg="No application specified !") + backupfile = pop_arg(args, msg="No backup file specified !") + _local_restore(appid, backupfile, not self.config.no_drop) + + +class DBCopyCommand(Command): + """Copy the system database of an application (backup and restore). + + + the identifier of the application to backup + format [[user@]host:]appname + + + the identifier of the application to restore + """ + name = 'db-copy' + arguments = ' ' + + options = ( + ("no-drop", + {'short': 'n', 'action' : 'store_true', + 'default' : False, + 'help': 'For some reason the database doesn\'t exist and so ' + 'should not be dropped.'} + ), + ("keep-dump", + {'short': 'k', 'action' : 'store_true', + 'default' : False, + 'help': 'Specify that the dump file should not be automatically removed.'} + ), + ('sudo', + {'short': 's', 'action' : 'store_true', + 'default' : False, + 'help': 'Use sudo on the remote host.'} + ), + ) + + def run(self, args): + import tempfile + srcappid = pop_arg(args, 1, msg="No source application specified !") + destappid = pop_arg(args, msg="No destination application specified !") + output = tempfile.mktemp() + if ':' in srcappid: + host, srcappid = srcappid.split(':') + _remote_dump(host, srcappid, output, self.config.sudo) + else: + _local_dump(srcappid, output) + _local_restore(destappid, output, not self.config.no_drop) + if self.config.keep_dump: + print 'you can get the dump file at', output + else: + os.remove(output) + + +class CheckRepositoryCommand(Command): + """Check integrity of the system database of an application. + + + the identifier of the application to check + """ + name = 'db-check' + arguments = '' + + options = ( + ("checks", + {'short': 'c', 'type' : "csv", 'metavar' : '', + 'default' : ('entities', 'relations', 'metadata', 'schema', 'text_index'), + 'help': 'Comma separated list of check to run. By default run all \ +checks, i.e. entities, relations, text_index and metadata.'} + ), + + ("autofix", + {'short': 'a', 'type' : "yn", 'metavar' : '', + 'default' : False, + 'help': 'Automatically correct integrity problems if this option \ +is set to "y" or "yes", else only display them'} + ), + ("reindex", + {'short': 'r', 'type' : "yn", 'metavar' : '', + 'default' : False, + 'help': 're-indexes the database for full text search if this \ +option is set to "y" or "yes" (may be long for large database).'} + ), + + ) + + def run(self, args): + from cubicweb.server.checkintegrity import check + appid = pop_arg(args, 1, msg="No application specified !") + config = ServerConfiguration.config_for(appid) + repo, cnx = repo_cnx(config) + check(repo, cnx, + self.config.checks, self.config.reindex, self.config.autofix) + + +class RebuildFTICommand(Command): + """Rebuild the full-text index of the system database of an application. + + + the identifier of the application to rebuild + """ + name = 'db-rebuild-fti' + arguments = '' + + options = () + + def run(self, args): + from cubicweb.server.checkintegrity import reindex_entities + appid = pop_arg(args, 1, msg="No application specified !") + config = ServerConfiguration.config_for(appid) + repo, cnx = repo_cnx(config) + session = repo._get_session(cnx.sessionid, setpool=True) + reindex_entities(repo.schema, session) + cnx.commit() + + +class SynchronizeApplicationSchemaCommand(Command): + """Synchronize persistent schema with cube schema. + + Will synchronize common stuff between the cube schema and the + actual persistent schema, but will not add/remove any entity or relation. + + + the identifier of the application to synchronize. + """ + name = 'schema-sync' + arguments = '' + + def run(self, args): + appid = pop_arg(args, msg="No application specified !") + config = ServerConfiguration.config_for(appid) + mih = config.migration_handler() + mih.cmd_synchronize_schema() + + +register_commands( (CreateApplicationDBCommand, + InitApplicationCommand, + GrantUserOnApplicationCommand, + StartRepositoryCommand, + DBDumpCommand, + DBRestoreCommand, + DBCopyCommand, + CheckRepositoryCommand, + RebuildFTICommand, + SynchronizeApplicationSchemaCommand, + ) )