"""a class implementing basic actions used in migration scripts.The following schema actions are supported for now:* add/drop/rename attribute* add/drop entity/relation type* rename entity typeThe following data actions are supported for now:* add an entity* execute raw RQL queries:organization: Logilab:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2.:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses"""from__future__importwith_statement__docformat__="restructuredtext en"importsysimportosimporttarfileimporttempfileimportshutilimportos.pathasospfromdatetimeimportdatetimefromglobimportglobfromcopyimportcopyfromwarningsimportwarnfromlogilab.common.deprecationimportdeprecatedfromlogilab.common.decoratorsimportcached,clear_cachefromlogilab.common.testlibimportmock_objectfromyams.constraintsimportSizeConstraintfromyams.schema2sqlimporteschema2sql,rschema2sqlfromcubicwebimportAuthenticationErrorfromcubicweb.schemaimport(META_RTYPES,VIRTUAL_RTYPES,CubicWebRelationSchema,order_eschemas)fromcubicweb.dbapiimportget_repository,repo_connectfromcubicweb.migrationimportMigrationHelper,yesfromcubicweb.server.sessionimporthooks_controltry:fromcubicweb.serverimportSOURCE_TYPES,schemaserialasssfromcubicweb.server.utilsimportmanager_userpasswd,ask_source_configfromcubicweb.server.sqlutilsimportsqlexec,SQL_PREFIXexceptImportError:# LAXpassclassServerMigrationHelper(MigrationHelper):"""specific migration helper for server side migration scripts, providind actions related to schema/data migration """def__init__(self,config,schema,interactive=True,repo=None,cnx=None,verbosity=1,connect=True):MigrationHelper.__init__(self,config,interactive,verbosity)ifnotinteractive:assertcnxassertrepoifcnxisnotNone:assertrepoself._cnx=cnxself.repo=repoelifconnect:self.repo_connect()# no config on shell to a remote instanceifconfigisnotNoneand(cnxorconnect):self.session.data['rebuild-infered']=Falseself.repo.hm.call_hooks('server_maintenance',repo=self.repo)ifnotschemaandnotgetattr(config,'quick_start',False):schema=config.load_schema(expand_cubes=True)self.fs_schema=schemaself._synchronized=set()# overriden from base MigrationHelper ######################################@cacheddefrepo_connect(self):self.repo=get_repository(method='inmemory',config=self.config)returnself.repodefcube_upgraded(self,cube,version):self.cmd_set_property('system.version.%s'%cube.lower(),unicode(version))self.commit()defshutdown(self):ifself.repoisnotNone:self.repo.shutdown()defmigrate(self,vcconf,toupgrade,options):ifnotoptions.fs_only:ifoptions.backup_dbisNone:self.backup_database()elifoptions.backup_db:self.backup_database(askconfirm=False)# disable notification during migrationwithhooks_control(self.session,self.session.HOOKS_ALLOW_ALL,'notification'):super(ServerMigrationHelper,self).migrate(vcconf,toupgrade,options)defcmd_process_script(self,migrscript,funcname=None,*args,**kwargs):"""execute a migration script in interactive mode, display the migration script path, ask for confirmation and execute it if confirmed """try:ifmigrscript.endswith('.sql'):ifself.execscript_confirm(migrscript):sqlexec(open(migrscript).read(),self.session.system_sql)elifmigrscript.endswith('.py')ormigrscript.endswith('.txt'):returnsuper(ServerMigrationHelper,self).cmd_process_script(migrscript,funcname,*args,**kwargs)else:printprint('-> ignoring %s, only .py .sql and .txt scripts are considered'%migrscript)printself.commit()except:self.rollback()raise# server specific migration methods ########################################defbackup_database(self,backupfile=None,askconfirm=True):config=self.configrepo=self.repo_connect()# pathstimestamp=datetime.now().strftime('%Y-%m-%d_%H-%M-%S')instbkdir=osp.join(config.appdatahome,'backup')ifnotosp.exists(instbkdir):os.makedirs(instbkdir)backupfile=backupfileorosp.join(instbkdir,'%s-%s.tar.gz'%(config.appid,timestamp))# check backup has to be doneifosp.exists(backupfile)andnot \self.confirm('Backup file %s exists, overwrite it?'%backupfile):print'-> no backup done.'returnelifaskconfirmandnotself.confirm('Backup %s database?'%config.appid):print'-> no backup done.'returnopen(backupfile,'w').close()# kinda lockos.chmod(backupfile,0600)# backuptmpdir=tempfile.mkdtemp(dir=instbkdir)try:forsourceinrepo.sources:try:source.backup(osp.join(tmpdir,source.uri),self.confirm)exceptException,ex:print'-> error trying to backup %s [%s]'%(source.uri,ex)ifnotself.confirm('Continue anyway?',default='n'):raiseSystemExit(1)else:breakelse:bkup=tarfile.open(backupfile,'w|gz')forfilenameinos.listdir(tmpdir):bkup.add(osp.join(tmpdir,filename),filename)bkup.close()# call hooksrepo.hm.call_hooks('server_backup',repo=repo,timestamp=timestamp)# doneprint'-> backup file',backupfilefinally:shutil.rmtree(tmpdir)defrestore_database(self,backupfile,drop=True,systemonly=True,askconfirm=True):# checkifnotosp.exists(backupfile):raiseException("Backup file %s doesn't exist"%backupfile)ifaskconfirmandnotself.confirm('Restore %s database from %s ?'%(self.config.appid,backupfile)):return# unpack backuptmpdir=tempfile.mkdtemp()try:bkup=tarfile.open(backupfile,'r|gz')excepttarfile.ReadError:# assume restoring old backupshutil.copy(backupfile,osp.join(tmpdir,'system'))else:fornameinbkup.getnames():ifname[0]in'/.':raiseException('Security check failed, path starts with "/" or "."')bkup.close()# XXX seek error if not close+open !?!bkup=tarfile.open(backupfile,'r|gz')bkup.extractall(path=tmpdir)bkup.close()self.config.open_connections_pools=Falserepo=self.repo_connect()forsourceinrepo.sources:ifsystemonlyandsource.uri!='system':continuetry:source.restore(osp.join(tmpdir,source.uri),self.confirm,drop)exceptException,exc:print'-> error trying to restore %s [%s]'%(source.uri,exc)ifnotself.confirm('Continue anyway?',default='n'):raiseSystemExit(1)shutil.rmtree(tmpdir)# call hooksrepo.open_connections_pools()repo.hm.call_hooks('server_restore',repo=repo,timestamp=backupfile)print'-> database restored.'@propertydefcnx(self):"""lazy connection"""try:returnself._cnxexceptAttributeError:sourcescfg=self.repo.config.sources()try:login=sourcescfg['admin']['login']pwd=sourcescfg['admin']['password']exceptKeyError:login,pwd=manager_userpasswd()whileTrue:try:self._cnx=repo_connect(self.repo,login,password=pwd)ifnot'managers'inself._cnx.user(self.session).groups:print'migration need an account in the managers group'else:breakexceptAuthenticationError:print'wrong user/password'except(KeyboardInterrupt,EOFError):print'aborting...'sys.exit(0)try:login,pwd=manager_userpasswd()except(KeyboardInterrupt,EOFError):print'aborting...'sys.exit(0)self.session.keep_pool_mode('transaction')self.session.data['rebuild-infered']=Falsereturnself._cnx@propertydefsession(self):ifself.configisnotNone:session=self.repo._get_session(self.cnx.sessionid)ifsession.poolisNone:session.set_read_security(False)session.set_write_security(False)session.set_pool()returnsession# no access to session on remote instancereturnNonedefcommit(self):ifhasattr(self,'_cnx'):self._cnx.commit()ifself.session:self.session.set_pool()defrollback(self):ifhasattr(self,'_cnx'):self._cnx.rollback()ifself.session:self.session.set_pool()defrqlexecall(self,rqliter,cachekey=None,ask_confirm=True):forrql,kwargsinrqliter:self.rqlexec(rql,kwargs,cachekey,ask_confirm=ask_confirm)@cacheddef_create_context(self):"""return a dictionary to use as migration script execution context"""context=super(ServerMigrationHelper,self)._create_context()context.update({'commit':self.checkpoint,'rollback':self.rollback,'checkpoint':deprecated('[3.6] use commit')(self.checkpoint),'sql':self.sqlexec,'rql':self.rqlexec,'rqliter':self.rqliter,'schema':self.repo.get_schema(),'cnx':self.cnx,'fsschema':self.fs_schema,'session':self.session,'repo':self.repo,'synchronize_schema':deprecated()(self.cmd_sync_schema_props_perms),# 3.4'synchronize_eschema':deprecated()(self.cmd_sync_schema_props_perms),# 3.4'synchronize_rschema':deprecated()(self.cmd_sync_schema_props_perms),# 3.4})returncontext@cacheddefgroup_mapping(self):"""cached group mapping"""returnss.group_mapping(self._cw)@cacheddefcstrtype_mapping(self):"""cached constraint types mapping"""returnss.cstrtype_mapping(self._cw)defexec_event_script(self,event,cubepath=None,funcname=None,*args,**kwargs):ifcubepath:apc=osp.join(cubepath,'migration','%s.py'%event)else:apc=osp.join(self.config.migration_scripts_dir(),'%s.py'%event)ifosp.exists(apc):ifself.config.free_wheel:self.cmd_deactivate_verification_hooks()self.info('executing %s',apc)confirm=self.confirmexecscript_confirm=self.execscript_confirmself.confirm=yesself.execscript_confirm=yestry:returnself.cmd_process_script(apc,funcname,*args,**kwargs)finally:self.confirm=confirmself.execscript_confirm=execscript_confirmifself.config.free_wheel:self.cmd_reactivate_verification_hooks()definstall_custom_sql_scripts(self,directory,driver):forfpathinglob(osp.join(directory,'*.sql.%s'%driver)):newname=osp.basename(fpath).replace('.sql.%s'%driver,'.%s.sql'%driver)warn('[3.5.6] rename %s into %s'%(fpath,newname),DeprecationWarning)print'-> installing',fpathsqlexec(open(fpath).read(),self.session.system_sql,False,delimiter=';;')forfpathinglob(osp.join(directory,'*.%s.sql'%driver)):print'-> installing',fpathsqlexec(open(fpath).read(),self.session.system_sql,False,delimiter=';;')# schema synchronization internals ########################################def_synchronize_permissions(self,erschema,teid):"""permission synchronization for an entity or relation type"""assertteid,erschemaif'update'inerschema.ACTIONSorerschema.final:# entity typeexprtype=u'ERQLExpression'else:# relation typeexprtype=u'RRQLExpression'gm=self.group_mapping()confirm=self.verbosity>=2# * remove possibly deprecated permission (eg in the persistent schema# but not in the new schema)# * synchronize existing expressions# * add new groups/expressionsforactioninerschema.ACTIONS:perm='%s_permission'%action# handle groupsnewgroups=list(erschema.get_groups(action))forgeid,gnameinself.rqlexec('Any G, GN WHERE T %s G, G name GN, ''T eid %%(x)s'%perm,{'x':teid},'x',ask_confirm=False):ifnotgnameinnewgroups:ifnotconfirmorself.confirm('Remove %s permission of %s to %s?'%(action,erschema,gname)):self.rqlexec('DELETE T %s G WHERE G eid %%(x)s, T eid %s'%(perm,teid),{'x':geid},'x',ask_confirm=False)else:newgroups.remove(gname)forgnameinnewgroups:ifnotconfirmorself.confirm('Grant %s permission of %s to %s?'%(action,erschema,gname)):self.rqlexec('SET T %s G WHERE G eid %%(x)s, T eid %s'%(perm,teid),{'x':gm[gname]},'x',ask_confirm=False)# handle rql expressionsnewexprs=dict((expr.expression,expr)forexprinerschema.get_rqlexprs(action))forexpreid,expressioninself.rqlexec('Any E, EX WHERE T %s E, E expression EX, ''T eid %s'%(perm,teid),ask_confirm=False):ifnotexpressioninnewexprs:ifnotconfirmorself.confirm('Remove %s expression for %s permission of %s?'%(expression,action,erschema)):# deleting the relation will delete the expression entityself.rqlexec('DELETE T %s E WHERE E eid %%(x)s, T eid %s'%(perm,teid),{'x':expreid},'x',ask_confirm=False)else:newexprs.pop(expression)forexpressioninnewexprs.values():expr=expression.expressionifnotconfirmorself.confirm('Add %s expression for %s permission of %s?'%(expr,action,erschema)):self.rqlexec('INSERT RQLExpression X: X exprtype %%(exprtype)s, ''X expression %%(expr)s, X mainvars %%(vars)s, T %s X ''WHERE T eid %%(x)s'%perm,{'expr':expr,'exprtype':exprtype,'vars':expression.mainvars,'x':teid},'x',ask_confirm=False)def_synchronize_rschema(self,rtype,syncrdefs=True,syncperms=True,syncprops=True):"""synchronize properties of the persistent relation schema against its current definition: * description * symmetric, meta * inlined * relation definitions if `syncrdefs` * permissions if `syncperms` physical schema changes should be handled by repository's schema hooks """rtype=str(rtype)ifrtypeinself._synchronized:returnself._synchronized.add(rtype)rschema=self.fs_schema.rschema(rtype)reporschema=self.repo.schema.rschema(rtype)ifsyncprops:assertreporschema.eid,reporschemaself.rqlexecall(ss.updaterschema2rql(rschema,reporschema.eid),ask_confirm=self.verbosity>=2)ifsyncrdefs:forsubj,objinrschema.rdefs:if(subj,obj)notinreporschema.rdefs:continueifrschemainVIRTUAL_RTYPES:continueself._synchronize_rdef_schema(subj,rschema,obj,syncprops=syncprops,syncperms=syncperms)def_synchronize_eschema(self,etype,syncperms=True):"""synchronize properties of the persistent entity schema against its current definition: * description * internationalizable, fulltextindexed, indexed, meta * relations from/to this entity * permissions if `syncperms` """etype=str(etype)ifetypeinself._synchronized:returnself._synchronized.add(etype)repoeschema=self.repo.schema.eschema(etype)try:eschema=self.fs_schema.eschema(etype)exceptKeyError:returnrepospschema=repoeschema.specializes()espschema=eschema.specializes()ifrepospschemaandnotespschema:self.rqlexec('DELETE X specializes Y WHERE X is CWEType, X name %(x)s',{'x':str(repoeschema)},ask_confirm=False)elifnotrepospschemaandespschema:self.rqlexec('SET X specializes Y WHERE X is CWEType, X name %(x)s, ''Y is CWEType, Y name %(y)s',{'x':str(repoeschema),'y':str(espschema)},ask_confirm=False)self.rqlexecall(ss.updateeschema2rql(eschema,repoeschema.eid),ask_confirm=self.verbosity>=2)forrschema,targettypes,roleineschema.relation_definitions(True):ifrschemainVIRTUAL_RTYPES:continueifrole=='subject':ifnotrschemainrepoeschema.subject_relations():continuesubjtypes,objtypes=[etype],targettypeselse:# role == 'object'ifnotrschemainrepoeschema.object_relations():continuesubjtypes,objtypes=targettypes,[etype]self._synchronize_rschema(rschema,syncperms=syncperms,syncrdefs=False)reporschema=self.repo.schema.rschema(rschema)forsubjinsubjtypes:forobjinobjtypes:if(subj,obj)notinreporschema.rdefs:continueself._synchronize_rdef_schema(subj,rschema,obj)ifsyncperms:self._synchronize_permissions(eschema,repoeschema.eid)def_synchronize_rdef_schema(self,subjtype,rtype,objtype,syncperms=True,syncprops=True):"""synchronize properties of the persistent relation definition schema against its current definition: * order and other properties * constraints """subjtype,objtype=str(subjtype),str(objtype)rschema=self.fs_schema.rschema(rtype)reporschema=self.repo.schema.rschema(rschema)if(subjtype,rschema,objtype)inself._synchronized:returnself._synchronized.add((subjtype,rschema,objtype))ifrschema.symmetric:self._synchronized.add((objtype,rschema,subjtype))rdef=rschema.rdef(subjtype,objtype)ifrdef.infered:return# don't try to synchronize infered relation defsrepordef=reporschema.rdef(subjtype,objtype)confirm=self.verbosity>=2ifsyncprops:# propertiesself.rqlexecall(ss.updaterdef2rql(rdef,repordef.eid),ask_confirm=confirm)# constraintsnewconstraints=list(rdef.constraints)# 1. remove old constraints and update constraints of the same type# NOTE: don't use rschema.constraint_by_type because it may be# out of sync with newconstraints when multiple# constraints of the same type are usedforcstrinrepordef.constraints:fornewcstrinnewconstraints:ifnewcstr.type()==cstr.type():breakelse:newcstr=NoneifnewcstrisNone:self.rqlexec('DELETE X constrained_by C WHERE C eid %(x)s',{'x':cstr.eid},'x',ask_confirm=confirm)else:newconstraints.remove(newcstr)value=unicode(newcstr.serialize())ifvalue!=unicode(cstr.serialize()):self.rqlexec('SET X value %(v)s WHERE X eid %(x)s',{'x':cstr.eid,'v':value},'x',ask_confirm=confirm)# 2. add new constraintscstrtype_map=self.cstrtype_mapping()self.rqlexecall(ss.constraints2rql(cstrtype_map,newconstraints,repordef.eid),ask_confirm=confirm)ifsyncpermsandnotrschemainVIRTUAL_RTYPES:self._synchronize_permissions(rdef,repordef.eid)# base actions ############################################################defcheckpoint(self,ask_confirm=True):"""checkpoint action"""ifnotask_confirmorself.confirm('Commit now ?',shell=False):self.commit()defcmd_add_cube(self,cube,update_database=True):self.cmd_add_cubes((cube,),update_database)defcmd_add_cubes(self,cubes,update_database=True):"""update_database is telling if the database schema should be updated or if only the relevant eproperty should be inserted (for the case where a cube has been extracted from an existing instance, so the cube schema is already in there) """newcubes=super(ServerMigrationHelper,self).cmd_add_cubes(cubes)ifnotnewcubes:returnforcubeinnewcubes:self.cmd_set_property('system.version.'+cube,self.config.cube_version(cube))ifcubeinSOURCE_TYPES:# don't use config.sources() in case some sources have been# disabled for migrationsourcescfg=self.config.read_sources_file()sourcescfg[cube]=ask_source_config(cube)self.config.write_sources_file(sourcescfg)clear_cache(self.config,'read_sources_file')# ensure added cube is in config cubes# XXX worth restoring on error?ifnotcubeinself.config._cubes:self.config._cubes+=(cube,)ifnotupdate_database:self.commit()returnnewcubes_schema=self.config.load_schema(construction_mode='non-strict')# XXX we have to replace fs_schema, used in cmd_add_relation_type# etc. and fsschema of migration script contextsself.fs_schema=self._create_context()['fsschema']=newcubes_schemanew=set()# execute pre-create filesdriver=self.repo.system_source.dbdriverforpackinreversed(newcubes):cubedir=self.config.cube_dir(pack)self.install_custom_sql_scripts(osp.join(cubedir,'schema'),driver)self.exec_event_script('precreate',cubedir)# add new entity and relation typesforrschemainnewcubes_schema.relations():ifnotrschemainself.repo.schema:self.cmd_add_relation_type(rschema.type)new.add(rschema.type)toadd=[eschemaforeschemainnewcubes_schema.entities()ifnoteschemainself.repo.schema]foreschemainorder_eschemas(toadd):self.cmd_add_entity_type(eschema.type)new.add(eschema.type)# check if attributes has been added to existing entitiesforrschemainnewcubes_schema.relations():existingschema=self.repo.schema.rschema(rschema.type)for(fromtype,totype)inrschema.rdefs:if(fromtype,totype)inexistingschema.rdefs:continue# check we should actually add the relation definitionifnot(fromtypeinnewortotypeinneworrschemainnew):continueself.cmd_add_relation_definition(str(fromtype),rschema.type,str(totype))# execute post-create filesforpackinreversed(newcubes):self.exec_event_script('postcreate',self.config.cube_dir(pack))self.commit()defcmd_remove_cube(self,cube,removedeps=False):removedcubes=super(ServerMigrationHelper,self).cmd_remove_cube(cube,removedeps)ifnotremovedcubes:returnfsschema=self.fs_schemaremovedcubes_schema=self.config.load_schema(construction_mode='non-strict')reposchema=self.repo.schema# execute pre-remove filesforpackinreversed(removedcubes):self.exec_event_script('preremove',self.config.cube_dir(pack))# remove cubes'entity and relation typesforrschemainfsschema.relations():ifnotrschemainremovedcubes_schemaandrschemainreposchema:self.cmd_drop_relation_type(rschema.type)toremove=[eschemaforeschemainfsschema.entities()ifnoteschemainremovedcubes_schemaandeschemainreposchema]foreschemainreversed(order_eschemas(toremove)):self.cmd_drop_entity_type(eschema.type)forrschemainfsschema.relations():ifrschemainremovedcubes_schemaandrschemainreposchema:# check if attributes/relations has been added to entities from# other cubesforfromtype,totypeinrschema.rdefs:if(fromtype,totype)notinremovedcubes_schema[rschema.type].rdefsand \(fromtype,totype)inreposchema[rschema.type].rdefs:self.cmd_drop_relation_definition(str(fromtype),rschema.type,str(totype))# execute post-remove filesforpackinreversed(removedcubes):self.exec_event_script('postremove',self.config.cube_dir(pack))self.rqlexec('DELETE CWProperty X WHERE X pkey %(pk)s',{'pk':u'system.version.'+pack},ask_confirm=False)self.commit()# schema migration actions ################################################defcmd_add_attribute(self,etype,attrname,attrtype=None,commit=True):"""add a new attribute on the given entity type"""ifattrtypeisNone:rschema=self.fs_schema.rschema(attrname)attrtype=rschema.objects(etype)[0]self.cmd_add_relation_definition(etype,attrname,attrtype,commit=commit)defcmd_drop_attribute(self,etype,attrname,commit=True):"""drop an existing attribute from the given entity type `attrname` is a string giving the name of the attribute to drop """rschema=self.repo.schema.rschema(attrname)attrtype=rschema.objects(etype)[0]self.cmd_drop_relation_definition(etype,attrname,attrtype,commit=commit)defcmd_rename_attribute(self,etype,oldname,newname,commit=True):"""rename an existing attribute of the given entity type `oldname` is a string giving the name of the existing attribute `newname` is a string giving the name of the renamed attribute """eschema=self.fs_schema.eschema(etype)attrtype=eschema.destination(newname)# have to commit this first step anyway to get the definition# actually in the schemaself.cmd_add_attribute(etype,newname,attrtype,commit=True)# skipp NULL values if the attribute is requiredrql='SET X %s VAL WHERE X is %s, X %s VAL'%(newname,etype,oldname)card=eschema.rdef(newname).cardinality[0]ifcard=='1':rql+=', NOT X %s NULL'%oldnameself.rqlexec(rql,ask_confirm=self.verbosity>=2)# XXX if both attributes fulltext indexed, should skip fti rebuild# XXX if old attribute was fti indexed but not the new one old value# won't be removed from the index (this occurs on other kind of# fulltextindexed change...)self.cmd_drop_attribute(etype,oldname,commit=commit)defcmd_add_entity_type(self,etype,auto=True,commit=True):"""register a new entity type in auto mode, automatically register entity's relation where the targeted type is known """instschema=self.repo.schemaassertnotetypeininstschema# # XXX (syt) plz explain: if we're adding an entity type, it should# # not be there...# eschema = instschema[etype]# if eschema.final:# instschema.del_entity_type(etype)# else:eschema=self.fs_schema.eschema(etype)confirm=self.verbosity>=2groupmap=self.group_mapping()cstrtypemap=self.cstrtype_mapping()# register the entity into CWETypeexecute=self._cw.executess.execschemarql(execute,eschema,ss.eschema2rql(eschema,groupmap))# add specializes relation if neededself.rqlexecall(ss.eschemaspecialize2rql(eschema),ask_confirm=confirm)# register entity's attributesforrschema,attrschemaineschema.attribute_definitions():# ignore those meta relations, they will be automatically addedifrschema.typeinMETA_RTYPES:continueifnotrschema.typeininstschema:# need to add the relation type and to commit to get it# actually in the schemaself.cmd_add_relation_type(rschema.type,False,commit=True)# register relation definitionrdef=self._get_rdef(rschema,eschema,eschema.destination(rschema))ss.execschemarql(execute,rdef,ss.rdef2rql(rdef,cstrtypemap,groupmap),)# take care to newly introduced base class# XXX some part of this should probably be under the "if auto" blockforspschemaineschema.specialized_by(recursive=False):try:instspschema=instschema[spschema]exceptKeyError:# specialized entity type not in schema, ignorecontinueifinstspschema.specializes()!=eschema:self.rqlexec('SET D specializes P WHERE D eid %(d)s, P name %(pn)s',{'d':instspschema.eid,'pn':eschema.type},ask_confirm=confirm)forrschema,tschemas,roleinspschema.relation_definitions(True):fortschemaintschemas:ifnottschemaininstschema:continueifrole=='subject':subjschema=spschemaobjschema=tschemaifrschema.finalandrschemaininstspschema.subjrels:# attribute already set, has_rdef would check if# it's of the same type, we don't want this so# simply skip herecontinueelifrole=='object':subjschema=tschemaobjschema=spschemaif(rschema.rdef(subjschema,objschema).inferedor(instschema.has_relation(rschema)and(subjschema,objschema)ininstschema[rschema].rdefs)):continueself.cmd_add_relation_definition(subjschema.type,rschema.type,objschema.type)ifauto:# we have commit here to get relation types actually in the schemaself.commit()added=[]forrschemaineschema.subject_relations():# attribute relation have already been processed and# 'owned_by'/'created_by' will be automatically addedifrschema.finalorrschema.typeinMETA_RTYPES:continuertypeadded=rschema.typeininstschemafortargetschemainrschema.objects(etype):# ignore relations where the targeted type is not in the# current instance schematargettype=targetschema.typeifnottargettypeininstschemaandtargettype!=etype:continueifnotrtypeadded:# need to add the relation type and to commit to get it# actually in the schemaadded.append(rschema.type)self.cmd_add_relation_type(rschema.type,False,commit=True)rtypeadded=True# register relation definition# remember this two avoid adding twice non symmetric relation# such as "Emailthread forked_from Emailthread"added.append((etype,rschema.type,targettype))rdef=self._get_rdef(rschema,eschema,targetschema)ss.execschemarql(execute,rdef,ss.rdef2rql(rdef,cstrtypemap,groupmap))forrschemaineschema.object_relations():ifrschema.typeinMETA_RTYPES:continuertypeadded=rschema.typeininstschemaorrschema.typeinaddedfortargetschemainrschema.subjects(etype):# ignore relations where the targeted type is not in the# current instance schematargettype=targetschema.type# don't check targettype != etype since in this case the# relation has already been added as a subject relationifnottargettypeininstschema:continueifnotrtypeadded:# need to add the relation type and to commit to get it# actually in the schemaself.cmd_add_relation_type(rschema.type,False,commit=True)rtypeadded=Trueelif(targettype,rschema.type,etype)inadded:continue# register relation definitionrdef=self._get_rdef(rschema,targetschema,eschema)ss.execschemarql(execute,rdef,ss.rdef2rql(rdef,cstrtypemap,groupmap))ifcommit:self.commit()defcmd_drop_entity_type(self,etype,commit=True):"""unregister an existing entity type This will trigger deletion of necessary relation types and definitions """# XXX what if we delete an entity type which is specialized by other types# unregister the entity from CWETypeself.rqlexec('DELETE CWEType X WHERE X name %(etype)s',{'etype':etype},ask_confirm=self.verbosity>=2)ifcommit:self.commit()defcmd_rename_entity_type(self,oldname,newname,commit=True):"""rename an existing entity type in the persistent schema `oldname` is a string giving the name of the existing entity type `newname` is a string giving the name of the renamed entity type """self.rqlexec('SET ET name %(newname)s WHERE ET is CWEType, ET name %(oldname)s',{'newname':unicode(newname),'oldname':oldname},ask_confirm=False)ifcommit:self.commit()defcmd_add_relation_type(self,rtype,addrdef=True,commit=True):"""register a new relation type named `rtype`, as described in the schema description file. `addrdef` is a boolean value; when True, it will also add all relations of the type just added found in the schema definition file. Note that it implies an intermediate "commit" which commits the relation type creation (but not the relation definitions themselves, for which committing depends on the `commit` argument value). """reposchema=self.repo.schemarschema=self.fs_schema.rschema(rtype)execute=self._cw.execute# register the relation into CWRType and insert necessary relation# definitionsss.execschemarql(execute,rschema,ss.rschema2rql(rschema,addrdef=False))ifaddrdef:self.commit()gmap=self.group_mapping()cmap=self.cstrtype_mapping()forrdefinrschema.rdefs.itervalues():ifnot(reposchema.has_entity(rdef.subject)andreposchema.has_entity(rdef.object)):continueself._set_rdef_eid(rdef)ss.execschemarql(execute,rdef,ss.rdef2rql(rdef,cmap,gmap))ifrtypeinMETA_RTYPES:# if the relation is in META_RTYPES, ensure we're adding it for# all entity types *in the persistent schema*, not only those in# the fs schemaforetypeinself.repo.schema.entities():ifnotetypeinself.fs_schema:# get sample object type and rpropertiesobjtypes=rschema.objects()assertlen(objtypes)==1,objtypesobjtype=objtypes[0]rdef=copy(rschema.rdef(rschema.subjects(objtype)[0],objtype))rdef.subject=etyperdef.rtype=self.repo.schema.rschema(rschema)rdef.object=self.repo.schema.rschema(objtype)ss.execschemarql(execute,rdef,ss.rdef2rql(rdef,cmap,gmap))ifcommit:self.commit()defcmd_drop_relation_type(self,rtype,commit=True):"""unregister an existing relation type"""# unregister the relation from CWRTypeself.rqlexec('DELETE CWRType X WHERE X name %r'%rtype,ask_confirm=self.verbosity>=2)ifcommit:self.commit()defcmd_rename_relation(self,oldname,newname,commit=True):"""rename an existing relation `oldname` is a string giving the name of the existing relation `newname` is a string giving the name of the renamed relation """self.cmd_add_relation_type(newname,commit=True)self.rqlexec('SET X %s Y WHERE X %s Y'%(newname,oldname),ask_confirm=self.verbosity>=2)self.cmd_drop_relation_type(oldname,commit=commit)defcmd_add_relation_definition(self,subjtype,rtype,objtype,commit=True):"""register a new relation definition, from its definition found in the schema definition file """rschema=self.fs_schema.rschema(rtype)ifnotrtypeinself.repo.schema:self.cmd_add_relation_type(rtype,addrdef=False,commit=True)execute=self._cw.executerdef=self._get_rdef(rschema,subjtype,objtype)ss.execschemarql(execute,rdef,ss.rdef2rql(rdef,self.cstrtype_mapping(),self.group_mapping()))ifcommit:self.commit()def_get_rdef(self,rschema,subjtype,objtype):returnself._set_rdef_eid(rschema.rdefs[(subjtype,objtype)])def_set_rdef_eid(self,rdef):forattrin('rtype','subject','object'):schemaobj=getattr(rdef,attr)ifgetattr(schemaobj,'eid',None)isNone:schemaobj.eid=self.repo.schema[schemaobj].eidassertschemaobj.eidisnotNonereturnrdefdefcmd_drop_relation_definition(self,subjtype,rtype,objtype,commit=True):"""unregister an existing relation definition"""rschema=self.repo.schema.rschema(rtype)# unregister the definition from CWAttribute or CWRelationifrschema.final:etype='CWAttribute'else:etype='CWRelation'rql=('DELETE %s X WHERE X from_entity FE, FE name "%s",''X relation_type RT, RT name "%s", X to_entity TE, TE name "%s"')self.rqlexec(rql%(etype,subjtype,rtype,objtype),ask_confirm=self.verbosity>=2)ifcommit:self.commit()defcmd_sync_schema_props_perms(self,ertype=None,syncperms=True,syncprops=True,syncrdefs=True,commit=True):"""synchronize the persistent schema against the current definition schema. It will synch common stuff between the definition schema and the actual persistent schema, it won't add/remove any entity or relation. """assertsyncpermsorsyncprops,'nothing to do'ifertypeisnotNone:ifisinstance(ertype,(tuple,list)):assertlen(ertype)==3,'not a relation definition'assertsyncprops,'can\'t update permission for a relation definition'self._synchronize_rdef_schema(ertype[0],ertype[1],ertype[2],syncperms=syncperms,syncprops=syncprops)else:erschema=self.repo.schema[ertype]ifisinstance(erschema,CubicWebRelationSchema):self._synchronize_rschema(erschema,syncperms=syncperms,syncprops=syncprops,syncrdefs=syncrdefs)elifsyncprops:self._synchronize_eschema(erschema,syncperms=syncperms)else:self._synchronize_permissions(self.fs_schema[ertype],erschema.eid)else:foretypeinself.repo.schema.entities():ifsyncprops:self._synchronize_eschema(etype,syncperms=syncperms)else:self._synchronize_permissions(self.fs_schema[etype],etype.eid)ifcommit:self.commit()defcmd_change_relation_props(self,subjtype,rtype,objtype,commit=True,**kwargs):"""change some properties of a relation definition you usually want to use sync_schema_props_perms instead. """assertkwargsrestriction=[]ifsubjtypeandsubjtype!='Any':restriction.append('X from_entity FE, FE name "%s"'%subjtype)ifobjtypeandobjtype!='Any':restriction.append('X to_entity TE, TE name "%s"'%objtype)ifrtypeandrtype!='Any':restriction.append('X relation_type RT, RT name "%s"'%rtype)assertrestrictionvalues=[]fork,vinkwargs.items():values.append('X %s%%(%s)s'%(k,k))ifisinstance(v,str):kwargs[k]=unicode(v)rql='SET %s WHERE %s'%(','.join(values),','.join(restriction))self.rqlexec(rql,kwargs,ask_confirm=self.verbosity>=2)ifcommit:self.commit()defcmd_set_size_constraint(self,etype,rtype,size,commit=True):"""set change size constraint of a string attribute if size is None any size constraint will be removed. you usually want to use sync_schema_props_perms instead. """oldvalue=Noneforconstrinself.repo.schema.eschema(etype).rdef(rtype).constraints:ifisinstance(constr,SizeConstraint):oldvalue=constr.maxifoldvalue==size:returnifoldvalueisNoneandnotsizeisNone:ceid=self.rqlexec('INSERT CWConstraint C: C value %(v)s, C cstrtype CT ''WHERE CT name "SizeConstraint"',{'v':SizeConstraint(size).serialize()},ask_confirm=self.verbosity>=2)[0][0]self.rqlexec('SET X constrained_by C WHERE X from_entity S, X relation_type R, ''S name "%s", R name "%s", C eid %s'%(etype,rtype,ceid),ask_confirm=self.verbosity>=2)elifnotoldvalueisNone:ifnotsizeisNone:self.rqlexec('SET C value %%(v)s WHERE X from_entity S, X relation_type R,''X constrained_by C, C cstrtype CT, CT name "SizeConstraint",''S name "%s", R name "%s"'%(etype,rtype),{'v':unicode(SizeConstraint(size).serialize())},ask_confirm=self.verbosity>=2)else:self.rqlexec('DELETE X constrained_by C WHERE X from_entity S, X relation_type R,''X constrained_by C, C cstrtype CT, CT name "SizeConstraint",''S name "%s", R name "%s"'%(etype,rtype),ask_confirm=self.verbosity>=2)# cleanup unused constraintsself.rqlexec('DELETE CWConstraint C WHERE NOT X constrained_by C')ifcommit:self.commit()@deprecated('[3.2] use sync_schema_props_perms(ertype, syncprops=False)')defcmd_synchronize_permissions(self,ertype,commit=True):self.cmd_sync_schema_props_perms(ertype,syncprops=False,commit=commit)# Workflows handling ######################################################defcmd_add_workflow(self,name,wfof,default=True,commit=False,**kwargs):""" create a new workflow and links it to entity types :type name: unicode :param name: name of the workflow :type wfof: string or list/tuple of strings :param wfof: entity type(s) having this workflow :type default: bool :param default: tells wether this is the default workflow for the specified entity type(s); set it to false in the case of a subworkflow :rtype: `Workflow` """wf=self.cmd_create_entity('Workflow',name=unicode(name),**kwargs)ifnotisinstance(wfof,(list,tuple)):wfof=(wfof,)foretypeinwfof:rset=self.rqlexec('SET X workflow_of ET WHERE X eid %(x)s, ET name %(et)s',{'x':wf.eid,'et':etype},'x',ask_confirm=False)assertrset,'unexistant entity type %s'%etypeifdefault:self.rqlexec('SET ET default_workflow X WHERE X eid %(x)s, ET name %(et)s',{'x':wf.eid,'et':etype},'x',ask_confirm=False)ifcommit:self.commit()returnwf# XXX remove once cmd_add_[state|transition] are removeddef_get_or_create_wf(self,etypes):ifnotisinstance(etypes,(list,tuple)):etypes=(etypes,)rset=self.rqlexec('Workflow X WHERE X workflow_of ET, ET name %(et)s',{'et':etypes[0]})ifrset:returnrset.get_entity(0,0)returnself.cmd_add_workflow('%s workflow'%';'.join(etypes),etypes)@deprecated('[3.5] use add_workflow and Workflow.add_state method',stacklevel=3)defcmd_add_state(self,name,stateof,initial=False,commit=False,**kwargs):"""method to ease workflow definition: add a state for one or more entity type(s) """wf=self._get_or_create_wf(stateof)state=wf.add_state(name,initial,**kwargs)ifcommit:self.commit()returnstate.eid@deprecated('[3.5] use add_workflow and Workflow.add_transition method',stacklevel=3)defcmd_add_transition(self,name,transitionof,fromstates,tostate,requiredgroups=(),conditions=(),commit=False,**kwargs):"""method to ease workflow definition: add a transition for one or more entity type(s), from one or more state and to a single state """wf=self._get_or_create_wf(transitionof)tr=wf.add_transition(name,fromstates,tostate,requiredgroups,conditions,**kwargs)ifcommit:self.commit()returntr.eid@deprecated('[3.5] use Transition.set_transition_permissions method',stacklevel=3)defcmd_set_transition_permissions(self,treid,requiredgroups=(),conditions=(),reset=True,commit=False):"""set or add (if `reset` is False) groups and conditions for a transition """tr=self._cw.entity_from_eid(treid)tr.set_transition_permissions(requiredgroups,conditions,reset)ifcommit:self.commit()@deprecated('[3.5] use entity.fire_transition("transition") or entity.change_state("state")',stacklevel=3)defcmd_set_state(self,eid,statename,commit=False):self._cw.entity_from_eid(eid).change_state(statename)ifcommit:self.commit()# CWProperty handling ######################################################defcmd_property_value(self,pkey):rql='Any V WHERE X is CWProperty, X pkey %(k)s, X value V'rset=self.rqlexec(rql,{'k':pkey},ask_confirm=False)returnrset[0][0]defcmd_set_property(self,pkey,value):value=unicode(value)try:prop=self.rqlexec('CWProperty X WHERE X pkey %(k)s',{'k':pkey},ask_confirm=False).get_entity(0,0)except:self.cmd_create_entity('CWProperty',pkey=unicode(pkey),value=value)else:self.rqlexec('SET X value %(v)s WHERE X pkey %(k)s',{'k':pkey,'v':value},ask_confirm=False)# other data migration commands ###########################################@propertydef_cw(self):session=self.sessionifsessionisnotNone:session.set_pool()returnsessionreturnself.cnx.request()defcmd_create_entity(self,etype,commit=False,**kwargs):"""add a new entity of the given type"""entity=self._cw.create_entity(etype,**kwargs)ifcommit:self.commit()returnentity@deprecated('[3.5] use create_entity',stacklevel=3)defcmd_add_entity(self,etype,*args,**kwargs):"""add a new entity of the given type"""returnself.cmd_create_entity(etype,*args,**kwargs).eiddefsqlexec(self,sql,args=None,ask_confirm=True):"""execute the given sql if confirmed should only be used for low level stuff undoable with existing higher level actions """ifnotask_confirmorself.confirm('Execute sql: %s ?'%sql):try:cu=self.session.system_sql(sql,args)except:ex=sys.exc_info()[1]ifself.confirm('Error: %s\nabort?'%ex):raisereturntry:returncu.fetchall()except:# no result to fetchreturndefrqlexec(self,rql,kwargs=None,cachekey=None,build_descr=True,ask_confirm=True):"""rql action"""ifnotisinstance(rql,(tuple,list)):rql=((rql,kwargs),)res=Noneexecute=self._cw.executeforrql,kwargsinrql:ifkwargs:msg='%s (%s)'%(rql,kwargs)else:msg=rqlifnotask_confirmorself.confirm('Execute rql: %s ?'%msg):try:res=execute(rql,kwargs,cachekey,build_descr=build_descr)exceptException,ex:ifself.confirm('Error: %s\nabort?'%ex):raisereturnresdefrqliter(self,rql,kwargs=None,ask_confirm=True):returnForRqlIterator(self,rql,None,ask_confirm)# broken db commands ######################################################defcmd_change_attribute_type(self,etype,attr,newtype,commit=True):"""low level method to change the type of an entity attribute. This is a quick hack which has some drawback: * only works when the old type can be changed to the new type by the underlying rdbms (eg using ALTER TABLE) * the actual schema won't be updated until next startup """rschema=self.repo.schema.rschema(attr)oldtype=rschema.objects(etype)[0]rdefeid=rschema.rproperty(etype,oldtype,'eid')sql=("UPDATE CWAttribute ""SET to_entity=(SELECT eid FROM CWEType WHERE name='%s')""WHERE eid=%s")%(newtype,rdefeid)self.sqlexec(sql,ask_confirm=False)dbhelper=self.repo.system_source.dbhelpersqltype=dbhelper.TYPE_MAPPING[newtype]sql='ALTER TABLE %s ALTER COLUMN %s TYPE %s'%(etype,attr,sqltype)self.sqlexec(sql,ask_confirm=False)ifcommit:self.commit()defcmd_add_entity_type_table(self,etype,commit=True):"""low level method to create the sql table for an existing entity. This may be useful on accidental desync between the repository schema and a sql database """dbhelper=self.repo.system_source.dbhelpertablesql=eschema2sql(dbhelper,self.repo.schema.eschema(etype),prefix=SQL_PREFIX)forsqlintablesql.split(';'):ifsql.strip():self.sqlexec(sql)ifcommit:self.commit()defcmd_add_relation_type_table(self,rtype,commit=True):"""low level method to create the sql table for an existing relation. This may be useful on accidental desync between the repository schema and a sql database """dbhelper=self.repo.system_source.dbhelpertablesql=rschema2sql(dbhelper,self.repo.schema.rschema(rtype))forsqlintablesql.split(';'):ifsql.strip():self.sqlexec(sql)ifcommit:self.commit()@deprecated("[3.7] use session.disable_hook_categories('integrity')")defcmd_deactivate_verification_hooks(self):self.session.disable_hook_categories('integrity')@deprecated("[3.7] use session.enable_hook_categories('integrity')")defcmd_reactivate_verification_hooks(self):self.session.enable_hook_categories('integrity')classForRqlIterator:"""specific rql iterator to make the loop skipable"""def__init__(self,helper,rql,kwargs,ask_confirm):self._h=helperself.rql=rqlself.kwargs=kwargsself.ask_confirm=ask_confirmself._rsetit=Nonedef__iter__(self):returnselfdefnext(self):ifself._rsetitisnotNone:returnself._rsetit.next()rql,kwargs=self.rql,self.kwargsifkwargs:msg='%s (%s)'%(rql,kwargs)else:msg=rqlifself.ask_confirm:ifnotself._h.confirm('Execute rql: %s ?'%msg):raiseStopIterationtry:rset=self._h._cw.execute(rql,kwargs)exceptException,ex:ifself._h.confirm('Error: %s\nabort?'%ex):raiseelse:raiseStopIterationself._rsetit=iter(rset)returnself._rsetit.next()