"""cubicweb-ctl commands and command handlers specific to the server.serverconfig:organization: Logilab:copyright: 2001-2009 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2.:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses"""__docformat__='restructuredtext en'importsysimportosfromlogilab.common.configurationimportConfigurationfromlogilab.common.clcommandsimportregister_commands,cmd_run,pop_argfromlogilab.common.shellutilsimportASKfromcubicwebimportAuthenticationError,ExecutionError,ConfigurationError,underline_titlefromcubicweb.toolsutilsimportCommand,CommandHandlerfromcubicweb.serverimportSOURCE_TYPESfromcubicweb.server.utilsimportask_source_configfromcubicweb.server.serverconfigimportUSER_OPTIONS,ServerConfiguration# utility functions ###########################################################defsource_cnx(source,dbname=None,special_privs=False,verbose=True):"""open and return a connection to the system database defined in the given server.serverconfig """fromgetpassimportgetpassfromlogilab.common.dbimportget_connectiondbhost=source['db-host']ifdbnameisNone:dbname=source['db-name']driver=source['db-driver']print'-> connecting to %s database %s@%s'%(driver,dbname,dbhostor'localhost'),ifnotverboseor(notspecial_privsandsource.get('db-user')):user=source['db-user']print'as',userifsource.get('db-password'):password=source['db-password']else:password=getpass('password: ')else:printifspecial_privs:print'WARNING'print'the user will need the following special access rights on the database:'printspecial_privsprintdefault_user=source.get('db-user',os.environ.get('USER',''))user=raw_input('Connect as user ? [%r]: '%default_user)user=userordefault_userifuser==source.get('db-user')andsource.get('db-password'):password=source['db-password']else:password=getpass('password: ')returnget_connection(driver,dbhost,dbname,user,password=password,port=source.get('db-port'))defsystem_source_cnx(source,dbms_system_base=False,special_privs='CREATE/DROP DATABASE',verbose=True):"""shortcut to get a connextion to the instance system database defined in the given config. If <dbms_system_base> is True, connect to the dbms system database instead (for task such as create/drop the instance database) """ifdbms_system_base:fromlogilab.common.adbhimportget_adv_func_helpersystem_db=get_adv_func_helper(source['db-driver']).system_database()returnsource_cnx(source,system_db,special_privs=special_privs,verbose=verbose)returnsource_cnx(source,special_privs=special_privs,verbose=verbose)def_db_sys_cnx(source,what,db=None,user=None,verbose=True):"""return a connection on the RDMS system table (to create/drop a user or a database """importlogilab.commonaslgpfromlogilab.common.adbhimportget_adv_func_helperlgp.USE_MX_DATETIME=Falsespecial_privs=''driver=source['db-driver']helper=get_adv_func_helper(driver)ifuserisnotNoneandhelper.users_support:special_privs+='%s USER'%whatifdbisnotNone:special_privs+=' %s DATABASE'%what# connect on the dbms system base to create our basecnx=system_source_cnx(source,True,special_privs=special_privs,verbose=verbose)# disable autocommit (isolation_level(1)) because DROP and# CREATE DATABASE can't be executed in a transactiontry:cnx.set_isolation_level(0)exceptAttributeError:# set_isolation_level() is psycopg specificpassreturncnxdefrepo_cnx(config):"""return a in-memory repository and a db api connection it"""fromcubicweb.dbapiimportin_memory_cnxfromcubicweb.server.utilsimportmanager_userpasswdtry:login=config.sources()['admin']['login']pwd=config.sources()['admin']['password']exceptKeyError:login,pwd=manager_userpasswd()whileTrue:try:returnin_memory_cnx(config,login,pwd)exceptAuthenticationError:print'-> Error: wrong user/password.'# reset cubes else we'll have an assertion error on next retryconfig._cubes=Nonelogin,pwd=manager_userpasswd()# repository specific command handlers ########################################classRepositoryCreateHandler(CommandHandler):cmdname='create'cfgname='repository'defbootstrap(self,cubes,inputlevel=0):"""create an instance by copying files from the given cube and by asking information necessary to build required configuration files """config=self.configprintunderline_title('Configuring the repository')config.input_config('email',inputlevel)ifconfig.pyro_enabled():config.input_config('pyro-server',inputlevel)print'\n'+underline_title('Configuring the sources')sourcesfile=config.sources_file()sconfig=Configuration(options=SOURCE_TYPES['native'].options)sconfig.adapter='native'sconfig.input_config(inputlevel=inputlevel)sourcescfg={'system':sconfig}forcubeincubes:# if a source is named as the cube containing it, we need the# source to use the cube, so add it.ifcubeinSOURCE_TYPES:sourcescfg[cube]=ask_source_config(cube,inputlevel)printwhileASK.confirm('Enter another source ?',default_is_yes=False):available=sorted(stypeforstypeinSOURCE_TYPESifnotstypeincubes)whileTrue:sourcetype=raw_input('source type (%s): '%', '.join(available))ifsourcetypeinavailable:breakprint'-> unknown source type, use one of the available types.'whileTrue:sourceuri=raw_input('source uri: ').strip()ifsourceuri!='admin'andsourceurinotinsourcescfg:breakprint'-> uri already used, choose another one.'sourcescfg[sourceuri]=ask_source_config(sourcetype)sourcemodule=SOURCE_TYPES[sourcetype].moduleifnotsourcemodule.startswith('cubicweb.'):# module names look like cubes.mycube.themodulesourcecube=SOURCE_TYPES[sourcetype].module.split('.',2)[1]# if the source adapter is coming from an external component,# ensure it's specified in used cubesifnotsourcecubeincubes:cubes.append(sourcecube)sconfig=Configuration(options=USER_OPTIONS)sconfig.input_config(inputlevel=inputlevel)sourcescfg['admin']=sconfigconfig.write_sources_file(sourcescfg)# remember selected cubes for later initialization of the databaseconfig.write_bootstrap_cubes_file(cubes)defpostcreate(self):ifASK.confirm('Run db-create to create the system database ?'):verbosity=(self.config.mode=='installed')and'y'or'n'cmd_run('db-create',self.config.appid,'--verbose=%s'%verbosity)else:print('-> nevermind, you can do it later with ''"cubicweb-ctl db-create %s".'%self.config.appid)classRepositoryDeleteHandler(CommandHandler):cmdname='delete'cfgname='repository'defcleanup(self):"""remove instance's configuration and database"""fromlogilab.common.adbhimportget_adv_func_helpersource=self.config.sources()['system']dbname=source['db-name']helper=get_adv_func_helper(source['db-driver'])ifASK.confirm('Delete database %s ?'%dbname):user=source['db-user']orNonecnx=_db_sys_cnx(source,'DROP DATABASE',user=user)cursor=cnx.cursor()try:cursor.execute('DROP DATABASE %s'%dbname)print'-> database %s dropped.'%dbname# XXX should check we are not connected as userifuserandhelper.users_supportand \ASK.confirm('Delete user %s ?'%user,default_is_yes=False):cursor.execute('DROP USER %s'%user)print'-> user %s dropped.'%usercnx.commit()except:cnx.rollback()raiseclassRepositoryStartHandler(CommandHandler):cmdname='start'cfgname='repository'defstart_server(self,ctlconf,debug):command=['cubicweb-ctl start-repository ']ifdebug:command.append('--debug')command.append(self.config.appid)os.system(' '.join(command))classRepositoryStopHandler(CommandHandler):cmdname='stop'cfgname='repository'defpoststop(self):"""if pyro is enabled, ensure the repository is correctly unregistered """ifself.config.pyro_enabled():fromcubicweb.server.repositoryimportpyro_unregisterpyro_unregister(self.config)# repository specific commands ################################################classCreateInstanceDBCommand(Command):"""Create the system database of an instance (run after 'create'). You will be prompted for a login / password to use to connect to the system database. The given user should have almost all rights on the database (ie a super user on the dbms allowed to create database, users, languages...). <instance> the identifier of the instance to initialize. """name='db-create'arguments='<instance>'options=(('create-db',{'short':'c','type':'yn','metavar':'<y or n>','default':True,'help':'create the database (yes by default)'}),('verbose',{'short':'v','type':'yn','metavar':'<verbose>','default':'n','help':'verbose mode: will ask all possible configuration questions',}),('automatic',{'short':'a','type':'yn','metavar':'<auto>','default':'n','help':'automatic mode: never ask and use default answer to every question',}),)defrun(self,args):"""run the command with its specific arguments"""fromlogilab.common.adbhimportget_adv_func_helperfromindexerimportget_indexerverbose=self.get('verbose')automatic=self.get('automatic')appid=pop_arg(args,msg='No instance specified !')config=ServerConfiguration.config_for(appid)create_db=self.config.create_dbsource=config.sources()['system']driver=source['db-driver']helper=get_adv_func_helper(driver)ifcreate_db:print'\n'+underline_title('Creating the system database')# connect on the dbms system base to create our basedbcnx=_db_sys_cnx(source,'CREATE DATABASE and / or USER',verbose=verbose)cursor=dbcnx.cursor()try:ifhelper.users_support:user=source['db-user']ifnothelper.user_exists(cursor,user)and(automaticor \ASK.confirm('Create db user %s ?'%user,default_is_yes=False)):helper.create_user(source['db-user'],source['db-password'])print'-> user %s created.'%userdbname=source['db-name']ifdbnameinhelper.list_databases(cursor):ifautomaticorASK.confirm('Database %s already exists -- do you want to drop it ?'%dbname):cursor.execute('DROP DATABASE %s'%dbname)else:returnifdbcnx.logged_user!=source['db-user']:helper.create_database(cursor,dbname,source['db-user'],source['db-encoding'])else:helper.create_database(cursor,dbname,encoding=source['db-encoding'])dbcnx.commit()print'-> database %s created.'%source['db-name']except:dbcnx.rollback()raisecnx=system_source_cnx(source,special_privs='LANGUAGE C',verbose=verbose)cursor=cnx.cursor()indexer=get_indexer(driver)indexer.init_extensions(cursor)# postgres specific stuffifdriver=='postgres':# install plpythonu/plpgsql language if not installed by the cubelangs=sys.platform=='win32'and('plpgsql',)or('plpythonu','plpgsql')forextlanginlangs:helper.create_language(cursor,extlang)cursor.close()cnx.commit()print'-> database for instance %s created and necessary extensions installed.'%appidprintifautomaticorASK.confirm('Run db-init to initialize the system database ?'):cmd_run('db-init',config.appid)else:print('-> nevermind, you can do it later with ''"cubicweb-ctl db-init %s".'%self.config.appid)classInitInstanceCommand(Command):"""Initialize the system database of an instance (run after 'db-create'). You will be prompted for a login / password to use to connect to the system database. The given user should have the create tables, and grant permissions. <instance> the identifier of the instance to initialize. """name='db-init'arguments='<instance>'options=(('drop',{'short':'d','action':'store_true','default':False,'help':'insert drop statements to remove previously existant \tables, indexes... (no by default)'}),)defrun(self,args):print'\n'+underline_title('Initializing the system database')fromcubicweb.serverimportinit_repositoryappid=pop_arg(args,msg='No instance specified !')config=ServerConfiguration.config_for(appid)init_repository(config,drop=self.config.drop)classGrantUserOnInstanceCommand(Command):"""Grant a database user on a repository system database. <instance> the identifier of the instance <user> the database's user requiring grant access """name='db-grant-user'arguments='<instance> <user>'options=(('set-owner',{'short':'o','type':'yn','metavar':'<yes or no>','default':False,'help':'Set the user as tables owner if yes (no by default).'}),)defrun(self,args):"""run the command with its specific arguments"""fromcubicweb.server.sqlutilsimportsqlexec,sqlgrantsappid=pop_arg(args,1,msg='No instance specified !')user=pop_arg(args,msg='No user specified !')config=ServerConfiguration.config_for(appid)source=config.sources()['system']set_owner=self.config.set_ownercnx=system_source_cnx(source,special_privs='GRANT')cursor=cnx.cursor()schema=config.load_schema()try:sqlexec(sqlgrants(schema,source['db-driver'],user,set_owner=set_owner),cursor)exceptException,ex:cnx.rollback()importtracebacktraceback.print_exc()print'-> an error occured:',exelse:cnx.commit()print'-> rights granted to %s on instance %s.'%(appid,user)classResetAdminPasswordCommand(Command):"""Reset the administrator password. <instance> the identifier of the instance """name='reset-admin-pwd'arguments='<instance>'defrun(self,args):"""run the command with its specific arguments"""fromcubicweb.server.sqlutilsimportsqlexec,SQL_PREFIXfromcubicweb.server.utilsimportcrypt_password,manager_userpasswdappid=pop_arg(args,1,msg='No instance specified !')config=ServerConfiguration.config_for(appid)sourcescfg=config.read_sources_file()try:adminlogin=sourcescfg['admin']['login']exceptKeyError:print'-> Error: could not get cubicweb administrator login.'sys.exit(1)cnx=source_cnx(sourcescfg['system'])cursor=cnx.cursor()_,passwd=manager_userpasswd(adminlogin,confirm=True,passwdmsg='new password for %s'%adminlogin)try:sqlexec("UPDATE %(sp)sCWUser SET %(sp)supassword='%(p)s' WHERE %(sp)slogin='%(l)s'"%{'sp':SQL_PREFIX,'p':crypt_password(passwd),'l':adminlogin},cursor,withpb=False)sconfig=Configuration(options=USER_OPTIONS)sconfig['login']=adminloginsconfig['password']=passwdsourcescfg['admin']=sconfigconfig.write_sources_file(sourcescfg)exceptException,ex:cnx.rollback()importtracebacktraceback.print_exc()print'-> an error occured:',exelse:cnx.commit()print'-> password reset, sources file regenerated.'classStartRepositoryCommand(Command):"""Start an CubicWeb RQL server for a given instance. The server will be accessible through pyro <instance> the identifier of the instance to initialize. """name='start-repository'arguments='<instance>'options=(('debug',{'short':'D','action':'store_true','help':'start server in debug mode.'}),)defrun(self,args):fromcubicweb.server.serverimportRepositoryServerappid=pop_arg(args,msg='No instance specified !')config=ServerConfiguration.config_for(appid)debug=self.config.debug# create the serverserver=RepositoryServer(config,debug)# go ! (don't daemonize in debug mode)pidfile=config['pid-file']# ensure the directory where the pid-file should be set exists (for# instance /var/run/cubicweb may be deleted on computer restart)piddir=os.path.dirname(pidfile)ifnotos.path.exists(piddir):os.makedirs(piddir)ifnotdebugandserver.daemonize(pidfile)==-1:returnuid=config['uid']ifuidisnotNone:try:uid=int(uid)exceptValueError:frompwdimportgetpwnamuid=getpwnam(uid).pw_uidos.setuid(uid)server.install_sig_handlers()server.connect(config['host'],0)server.run()def_remote_dump(host,appid,output,sudo=False):# XXX generate unique/portable file namefromdatetimeimportdatefilename='%s-%s.tgz'%(appid,date.today().strftime('%Y-%m-%d'))dmpcmd='cubicweb-ctl db-dump -o /tmp/%s%s'%(filename,appid)ifsudo:dmpcmd='sudo %s'%(dmpcmd)dmpcmd='ssh -t %s "%s"'%(host,dmpcmd)printdmpcmdifos.system(dmpcmd):raiseExecutionError('Error while dumping the database')ifoutputisNone:output=filenamecmd='scp %s:/tmp/%s%s'%(host,filename,output)printcmdifos.system(cmd):raiseExecutionError('Error while retrieving the dump at /tmp/%s'%filename)rmcmd='ssh -t %s "rm -f /tmp/%s"'%(host,filename)printrmcmdifos.system(rmcmd)andnotASK.confirm('An error occured while deleting remote dump at /tmp/%s. ''Continue anyway?'%filename):raiseExecutionError('Error while deleting remote dump at /tmp/%s'%filename)def_local_dump(appid,output):config=ServerConfiguration.config_for(appid)# schema=1 to avoid unnecessary schema loadingmih=config.migration_handler(connect=False,schema=1,verbosity=1)mih.backup_database(output,askconfirm=False)mih.shutdown()def_local_restore(appid,backupfile,drop,systemonly=True):config=ServerConfiguration.config_for(appid)config.verbosity=1# else we won't be asked for confirmation on problemsconfig.repairing=1# don't check versions# schema=1 to avoid unnecessary schema loadingmih=config.migration_handler(connect=False,schema=1,verbosity=1)mih.restore_database(backupfile,drop,systemonly,askconfirm=False)repo=mih.repo_connect()# version of the databasedbversions=repo.get_versions()mih.shutdown()ifnotdbversions:print"bad or missing version information in the database, don't upgrade file system"return# version of installed softwareeversion=dbversions['cubicweb']status=instance_status(config,eversion,dbversions)# * database version > installed softwareifstatus=='needsoftupgrade':print"** The database of %s is more recent than the installed software!"%config.appidprint"** Upgrade your software, then migrate the database by running the command"print"** 'cubicweb-ctl upgrade %s'"%config.appidreturn# * database version < installed software, an upgrade will be necessary# anyway, just rewrite vc.conf and warn user he has to upgradeelifstatus=='needapplupgrade':print"** The database of %s is older than the installed software."%config.appidprint"** Migrate the database by running the command"print"** 'cubicweb-ctl upgrade %s'"%config.appidreturn# * database version = installed software, database version = instance fs version# ok!definstance_status(config,cubicwebapplversion,vcconf):cubicwebversion=config.cubicweb_version()ifcubicwebapplversion>cubicwebversion:return'needsoftupgrade'ifcubicwebapplversion<cubicwebversion:return'needapplupgrade'forcubeinconfig.cubes():try:softversion=config.cube_version(cube)exceptConfigurationError:print'-> Error: no cube version information for %s, please check that the cube is installed.'%cubecontinuetry:applversion=vcconf[cube]exceptKeyError:print'-> Error: no cube version information for %s in version configuration.'%cubecontinueifsoftversion==applversion:continueifsoftversion>applversion:return'needsoftupgrade'elifsoftversion<applversion:return'needapplupgrade'returnNoneclassDBDumpCommand(Command):"""Backup the system database of an instance. <instance> the identifier of the instance to backup format [[user@]host:]appname """name='db-dump'arguments='<instance>'options=(('output',{'short':'o','type':'string','metavar':'<file>','default':None,'help':'Specify the backup file where the backup will be stored.'}),('sudo',{'short':'s','action':'store_true','default':False,'help':'Use sudo on the remote host.'}),)defrun(self,args):appid=pop_arg(args,1,msg='No instance specified !')if':'inappid:host,appid=appid.split(':')_remote_dump(host,appid,self.config.output,self.config.sudo)else:_local_dump(appid,self.config.output)classDBRestoreCommand(Command):"""Restore the system database of an instance. <instance> the identifier of the instance to restore """name='db-restore'arguments='<instance> <backupfile>'options=(('no-drop',{'short':'n','action':'store_true','default':False,'help':'for some reason the database doesn\'t exist and so ''should not be dropped.'}),('restore-all',{'short':'r','action':'store_true','default':False,'help':'restore everything, eg not only the system source database ''but also data for all sources supporting backup/restore and custom ''instance data. In that case, <backupfile> is expected to be the ''timestamp of the backup to restore, not a file'}),)defrun(self,args):appid=pop_arg(args,1,msg='No instance specified !')backupfile=pop_arg(args,msg='No backup file or timestamp specified !')_local_restore(appid,backupfile,drop=notself.config.no_drop,systemonly=notself.config.restore_all)classDBCopyCommand(Command):"""Copy the system database of an instance (backup and restore). <src-instance> the identifier of the instance to backup format [[user@]host:]appname <dest-instance> the identifier of the instance to restore """name='db-copy'arguments='<src-instance> <dest-instance>'options=(('no-drop',{'short':'n','action':'store_true','default':False,'help':'For some reason the database doesn\'t exist and so ''should not be dropped.'}),('keep-dump',{'short':'k','action':'store_true','default':False,'help':'Specify that the dump file should not be automatically removed.'}),('sudo',{'short':'s','action':'store_true','default':False,'help':'Use sudo on the remote host.'}),)defrun(self,args):importtempfilesrcappid=pop_arg(args,1,msg='No source instance specified !')destappid=pop_arg(args,msg='No destination instance specified !')output=tempfile.mkstemp()[1]if':'insrcappid:host,srcappid=srcappid.split(':')_remote_dump(host,srcappid,output,self.config.sudo)else:_local_dump(srcappid,output)_local_restore(destappid,output,notself.config.no_drop)ifself.config.keep_dump:print'-> you can get the dump file at',outputelse:os.remove(output)classCheckRepositoryCommand(Command):"""Check integrity of the system database of an instance. <instance> the identifier of the instance to check """name='db-check'arguments='<instance>'options=(('checks',{'short':'c','type':'csv','metavar':'<check list>','default':('entities','relations','metadata','schema','text_index'),'help':'Comma separated list of check to run. By default run all \checks, i.e. entities, relations, text_index and metadata.'}),('autofix',{'short':'a','type':'yn','metavar':'<yes or no>','default':False,'help':'Automatically correct integrity problems if this option \is set to "y" or "yes", else only display them'}),('reindex',{'short':'r','type':'yn','metavar':'<yes or no>','default':False,'help':'re-indexes the database for full text search if this \option is set to "y" or "yes" (may be long for large database).'}),('force',{'short':'f','action':'store_true','default':False,'help':'don\'t check instance is up to date.'}),)defrun(self,args):fromcubicweb.server.checkintegrityimportcheckappid=pop_arg(args,1,msg='No instance specified !')config=ServerConfiguration.config_for(appid)config.repairing=self.config.forcerepo,cnx=repo_cnx(config)check(repo,cnx,self.config.checks,self.config.reindex,self.config.autofix)classRebuildFTICommand(Command):"""Rebuild the full-text index of the system database of an instance. <instance> the identifier of the instance to rebuild """name='db-rebuild-fti'arguments='<instance>'options=()defrun(self,args):fromcubicweb.server.checkintegrityimportreindex_entitiesappid=pop_arg(args,1,msg='No instance specified !')config=ServerConfiguration.config_for(appid)repo,cnx=repo_cnx(config)session=repo._get_session(cnx.sessionid,setpool=True)reindex_entities(repo.schema,session)cnx.commit()classSynchronizeInstanceSchemaCommand(Command):"""Synchronize persistent schema with cube schema. Will synchronize common stuff between the cube schema and the actual persistent schema, but will not add/remove any entity or relation. <instance> the identifier of the instance to synchronize. """name='schema-sync'arguments='<instance>'defrun(self,args):appid=pop_arg(args,msg='No instance specified !')config=ServerConfiguration.config_for(appid)mih=config.migration_handler()mih.cmd_synchronize_schema()register_commands((CreateInstanceDBCommand,InitInstanceCommand,GrantUserOnInstanceCommand,ResetAdminPasswordCommand,StartRepositoryCommand,DBDumpCommand,DBRestoreCommand,DBCopyCommand,CheckRepositoryCommand,RebuildFTICommand,SynchronizeInstanceSchemaCommand,))