Added tag cubicweb-version-3.5.1 for changeset 77ed72f3c260
"""a class implementing basic actions used in migration scripts.The following schema actions are supported for now:* add/drop/rename attribute* add/drop entity/relation type* rename entity typeThe following data actions are supported for now:* add an entity* execute raw RQL queries:organization: Logilab:copyright: 2001-2009 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2.:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses"""__docformat__="restructuredtext en"importsysimportosimporttarfileimporttempfileimportshutilimportos.pathasospfromdatetimeimportdatetimefromlogilab.common.deprecationimportdeprecatedfromlogilab.common.decoratorsimportcached,clear_cachefromlogilab.common.adbhimportget_adv_func_helperfromyams.constraintsimportSizeConstraintfromyams.schema2sqlimporteschema2sql,rschema2sqlfromcubicwebimportAuthenticationError,ETYPE_NAME_MAPfromcubicweb.schemaimport(META_RTYPES,VIRTUAL_RTYPES,CubicWebRelationSchema,order_eschemas)fromcubicweb.dbapiimportget_repository,repo_connectfromcubicweb.common.migrationimportMigrationHelper,yestry:fromcubicweb.serverimportSOURCE_TYPES,schemaserialasssfromcubicweb.server.utilsimportmanager_userpasswd,ask_source_configfromcubicweb.server.sqlutilsimportsqlexec,SQL_PREFIXexceptImportError:# LAXpassclassServerMigrationHelper(MigrationHelper):"""specific migration helper for server side migration scripts, providind actions related to schema/data migration """def__init__(self,config,schema,interactive=True,repo=None,cnx=None,verbosity=1,connect=True):MigrationHelper.__init__(self,config,interactive,verbosity)ifnotinteractive:assertcnxassertrepoifcnxisnotNone:assertrepoself._cnx=cnxself.repo=repoself.session.data['rebuild-infered']=Falseelifconnect:self.repo_connect()ifnotschema:schema=config.load_schema(expand_cubes=True)self.fs_schema=schemaself._synchronized=set()# overriden from base MigrationHelper ######################################@cacheddefrepo_connect(self):self.repo=get_repository(method='inmemory',config=self.config)returnself.repodefcube_upgraded(self,cube,version):self.cmd_set_property('system.version.%s'%cube.lower(),unicode(version))self.commit()defshutdown(self):ifself.repoisnotNone:self.repo.shutdown()defmigrate(self,vcconf,toupgrade,options):ifnotoptions.fs_only:ifoptions.backup_dbisNone:self.backup_database()elifoptions.backup_db:self.backup_database(askconfirm=False)super(ServerMigrationHelper,self).migrate(vcconf,toupgrade,options)defprocess_script(self,migrscript,funcname=None,*args,**kwargs):"""execute a migration script in interactive mode, display the migration script path, ask for confirmation and execute it if confirmed """try:ifmigrscript.endswith('.sql'):ifself.execscript_confirm(migrscript):sqlexec(open(migrscript).read(),self.session.system_sql)else:returnsuper(ServerMigrationHelper,self).process_script(migrscript,funcname,*args,**kwargs)self.commit()except:self.rollback()raise# server specific migration methods ########################################defbackup_database(self,backupfile=None,askconfirm=True):config=self.configrepo=self.repo_connect()# pathstimestamp=datetime.now().strftime('%Y-%m-%d_%H-%M-%S')instbkdir=osp.join(config.appdatahome,'backup')ifnotosp.exists(instbkdir):os.makedirs(instbkdir)backupfile=backupfileorosp.join(instbkdir,'%s-%s.tar.gz'%(config.appid,timestamp))# check backup has to be doneifosp.exists(backupfile)andnot \self.confirm('Backup file %s exists, overwrite it?'%backupfile):print'-> no backup done.'returnelifaskconfirmandnotself.confirm('Backup %s database?'%config.appid):print'-> no backup done.'returnopen(backupfile,'w').close()# kinda lockos.chmod(backupfile,0600)# backuptmpdir=tempfile.mkdtemp(dir=instbkdir)try:forsourceinrepo.sources:try:source.backup(osp.join(tmpdir,source.uri))exceptException,exc:print'-> error trying to backup [%s]'%excifnotself.confirm('Continue anyway?',default='n'):raiseSystemExit(1)else:breakelse:bkup=tarfile.open(backupfile,'w|gz')forfilenameinos.listdir(tmpdir):bkup.add(osp.join(tmpdir,filename),filename)bkup.close()# call hooksrepo.hm.call_hooks('server_backup',repo=repo,timestamp=timestamp)# doneprint'-> backup file',backupfilefinally:shutil.rmtree(tmpdir)defrestore_database(self,backupfile,drop=True,systemonly=True,askconfirm=True):# checkifnotosp.exists(backupfile):raiseException("Backup file %s doesn't exist"%backupfile)returnifaskconfirmandnotself.confirm('Restore %s database from %s ?'%(self.config.appid,backupfile)):return# unpack backuptmpdir=tempfile.mkdtemp()try:bkup=tarfile.open(backupfile,'r|gz')excepttarfile.ReadError:# assume restoring old backupshutil.copy(backupfile,osp.join(tmpdir,'system'))else:fornameinbkup.getnames():ifname[0]in'/.':raiseException('Security check failed, path starts with "/" or "."')bkup.close()# XXX seek error if not close+open !?!bkup=tarfile.open(backupfile,'r|gz')bkup.extractall(path=tmpdir)bkup.close()self.config.open_connections_pools=Falserepo=self.repo_connect()forsourceinrepo.sources:ifsystemonlyandsource.uri!='system':continuetry:source.restore(osp.join(tmpdir,source.uri),drop=drop)exceptException,exc:print'-> error trying to restore [%s]'%excifnotself.confirm('Continue anyway?',default='n'):raiseSystemExit(1)shutil.rmtree(tmpdir)# call hooksrepo.open_connections_pools()repo.hm.call_hooks('server_restore',repo=repo,timestamp=backupfile)print'-> database restored.'@propertydefcnx(self):"""lazy connection"""try:returnself._cnxexceptAttributeError:sourcescfg=self.repo.config.sources()try:login=sourcescfg['admin']['login']pwd=sourcescfg['admin']['password']exceptKeyError:login,pwd=manager_userpasswd()whileTrue:try:self._cnx=repo_connect(self.repo,login,pwd)ifnot'managers'inself._cnx.user(self.session).groups:print'migration need an account in the managers group'else:breakexceptAuthenticationError:print'wrong user/password'except(KeyboardInterrupt,EOFError):print'aborting...'sys.exit(0)try:login,pwd=manager_userpasswd()except(KeyboardInterrupt,EOFError):print'aborting...'sys.exit(0)self.session.keep_pool_mode('transaction')self.session.data['rebuild-infered']=Falsereturnself._cnx@propertydefsession(self):returnself.repo._get_session(self.cnx.sessionid)defcommit(self):ifhasattr(self,'_cnx'):self._cnx.commit()defrollback(self):ifhasattr(self,'_cnx'):self._cnx.rollback()defrqlexecall(self,rqliter,cachekey=None,ask_confirm=True):forrql,kwargsinrqliter:self.rqlexec(rql,kwargs,cachekey,ask_confirm)@cacheddef_create_context(self):"""return a dictionary to use as migration script execution context"""context=super(ServerMigrationHelper,self)._create_context()context.update({'checkpoint':self.checkpoint,'sql':self.sqlexec,'rql':self.rqlexec,'rqliter':self.rqliter,'schema':self.repo.schema,'fsschema':self.fs_schema,'session':self.session,'repo':self.repo,'synchronize_schema':deprecated()(self.cmd_sync_schema_props_perms),'synchronize_eschema':deprecated()(self.cmd_sync_schema_props_perms),'synchronize_rschema':deprecated()(self.cmd_sync_schema_props_perms),})returncontext@cacheddefgroup_mapping(self):"""cached group mapping"""self.session.set_pool()returnss.group_mapping(self.session)defexec_event_script(self,event,cubepath=None,funcname=None,*args,**kwargs):ifcubepath:apc=osp.join(cubepath,'migration','%s.py'%event)else:apc=osp.join(self.config.migration_scripts_dir(),'%s.py'%event)ifosp.exists(apc):ifself.config.free_wheel:fromcubicweb.server.hooksimportsetowner_after_add_entityself.repo.hm.unregister_hook(setowner_after_add_entity,'after_add_entity','')self.deactivate_verification_hooks()self.info('executing %s',apc)confirm=self.confirmexecscript_confirm=self.execscript_confirmself.confirm=yesself.execscript_confirm=yestry:returnself.process_script(apc,funcname,*args,**kwargs)finally:self.confirm=confirmself.execscript_confirm=execscript_confirmifself.config.free_wheel:self.repo.hm.register_hook(setowner_after_add_entity,'after_add_entity','')self.reactivate_verification_hooks()# schema synchronization internals ########################################def_synchronize_permissions(self,ertype):"""permission synchronization for an entity or relation type"""ifertypeinVIRTUAL_RTYPES:returnnewrschema=self.fs_schema[ertype]teid=self.repo.schema[ertype].eidif'update'innewrschema.ACTIONSornewrschema.is_final():# entity typeexprtype=u'ERQLExpression'else:# relation typeexprtype=u'RRQLExpression'assertteid,ertypegm=self.group_mapping()confirm=self.verbosity>=2# * remove possibly deprecated permission (eg in the persistent schema# but not in the new schema)# * synchronize existing expressions# * add new groups/expressionsforactioninnewrschema.ACTIONS:perm='%s_permission'%action# handle groupsnewgroups=list(newrschema.get_groups(action))forgeid,gnameinself.rqlexec('Any G, GN WHERE T %s G, G name GN, ''T eid %%(x)s'%perm,{'x':teid},'x',ask_confirm=False):ifnotgnameinnewgroups:ifnotconfirmorself.confirm('remove %s permission of %s to %s?'%(action,ertype,gname)):self.rqlexec('DELETE T %s G WHERE G eid %%(x)s, T eid %s'%(perm,teid),{'x':geid},'x',ask_confirm=False)else:newgroups.remove(gname)forgnameinnewgroups:ifnotconfirmorself.confirm('grant %s permission of %s to %s?'%(action,ertype,gname)):self.rqlexec('SET T %s G WHERE G eid %%(x)s, T eid %s'%(perm,teid),{'x':gm[gname]},'x',ask_confirm=False)# handle rql expressionsnewexprs=dict((expr.expression,expr)forexprinnewrschema.get_rqlexprs(action))forexpreid,expressioninself.rqlexec('Any E, EX WHERE T %s E, E expression EX, ''T eid %s'%(perm,teid),ask_confirm=False):ifnotexpressioninnewexprs:ifnotconfirmorself.confirm('remove %s expression for %s permission of %s?'%(expression,action,ertype)):# deleting the relation will delete the expression entityself.rqlexec('DELETE T %s E WHERE E eid %%(x)s, T eid %s'%(perm,teid),{'x':expreid},'x',ask_confirm=False)else:newexprs.pop(expression)forexpressioninnewexprs.values():expr=expression.expressionifnotconfirmorself.confirm('add %s expression for %s permission of %s?'%(expr,action,ertype)):self.rqlexec('INSERT RQLExpression X: X exprtype %%(exprtype)s, ''X expression %%(expr)s, X mainvars %%(vars)s, T %s X ''WHERE T eid %%(x)s'%perm,{'expr':expr,'exprtype':exprtype,'vars':expression.mainvars,'x':teid},'x',ask_confirm=False)def_synchronize_rschema(self,rtype,syncrdefs=True,syncperms=True):"""synchronize properties of the persistent relation schema against its current definition: * description * symetric, meta * inlined * relation definitions if `syncrdefs` * permissions if `syncperms` physical schema changes should be handled by repository's schema hooks """rtype=str(rtype)ifrtypeinself._synchronized:returnself._synchronized.add(rtype)rschema=self.fs_schema.rschema(rtype)self.rqlexecall(ss.updaterschema2rql(rschema),ask_confirm=self.verbosity>=2)reporschema=self.repo.schema.rschema(rtype)ifsyncrdefs:forsubj,objinrschema.iter_rdefs():ifnotreporschema.has_rdef(subj,obj):continueself._synchronize_rdef_schema(subj,rschema,obj)ifsyncperms:self._synchronize_permissions(rtype)def_synchronize_eschema(self,etype,syncperms=True):"""synchronize properties of the persistent entity schema against its current definition: * description * internationalizable, fulltextindexed, indexed, meta * relations from/to this entity * permissions if `syncperms` """etype=str(etype)ifetypeinself._synchronized:returnself._synchronized.add(etype)repoeschema=self.repo.schema.eschema(etype)try:eschema=self.fs_schema.eschema(etype)exceptKeyError:returnrepospschema=repoeschema.specializes()espschema=eschema.specializes()ifrepospschemaandnotespschema:self.rqlexec('DELETE X specializes Y WHERE X is CWEType, X name %(x)s',{'x':str(repoeschema)},ask_confirm=False)elifnotrepospschemaandespschema:self.rqlexec('SET X specializes Y WHERE X is CWEType, X name %(x)s, ''Y is CWEType, Y name %(y)s',{'x':str(repoeschema),'y':str(espschema)},ask_confirm=False)self.rqlexecall(ss.updateeschema2rql(eschema),ask_confirm=self.verbosity>=2)forrschema,targettypes,roleineschema.relation_definitions(True):ifrole=='subject':ifnotrschemainrepoeschema.subject_relations():continuesubjtypes,objtypes=[etype],targettypeselse:# role == 'object'ifnotrschemainrepoeschema.object_relations():continuesubjtypes,objtypes=targettypes,[etype]self._synchronize_rschema(rschema,syncperms=syncperms,syncrdefs=False)reporschema=self.repo.schema.rschema(rschema)forsubjinsubjtypes:forobjinobjtypes:ifnotreporschema.has_rdef(subj,obj):continueself._synchronize_rdef_schema(subj,rschema,obj)ifsyncperms:self._synchronize_permissions(etype)def_synchronize_rdef_schema(self,subjtype,rtype,objtype):"""synchronize properties of the persistent relation definition schema against its current definition: * order and other properties * constraints """subjtype,objtype=str(subjtype),str(objtype)rschema=self.fs_schema.rschema(rtype)reporschema=self.repo.schema.rschema(rschema)if(subjtype,rschema,objtype)inself._synchronized:returnself._synchronized.add((subjtype,rschema,objtype))ifrschema.symetric:self._synchronized.add((objtype,rschema,subjtype))confirm=self.verbosity>=2# propertiesself.rqlexecall(ss.updaterdef2rql(rschema,subjtype,objtype),ask_confirm=confirm)# constraintsnewconstraints=list(rschema.rproperty(subjtype,objtype,'constraints'))# 1. remove old constraints and update constraints of the same type# NOTE: don't use rschema.constraint_by_type because it may be# out of sync with newconstraints when multiple# constraints of the same type are usedforcstrinreporschema.rproperty(subjtype,objtype,'constraints'):fornewcstrinnewconstraints:ifnewcstr.type()==cstr.type():breakelse:newcstr=NoneifnewcstrisNone:self.rqlexec('DELETE X constrained_by C WHERE C eid %(x)s',{'x':cstr.eid},'x',ask_confirm=confirm)self.rqlexec('DELETE CWConstraint C WHERE C eid %(x)s',{'x':cstr.eid},'x',ask_confirm=confirm)else:newconstraints.remove(newcstr)values={'x':cstr.eid,'v':unicode(newcstr.serialize())}self.rqlexec('SET X value %(v)s WHERE X eid %(x)s',values,'x',ask_confirm=confirm)# 2. add new constraintsfornewcstrinnewconstraints:self.rqlexecall(ss.constraint2rql(rschema,subjtype,objtype,newcstr),ask_confirm=confirm)# base actions ############################################################defcheckpoint(self):"""checkpoint action"""ifself.confirm('commit now ?',shell=False):self.commit()defcmd_add_cube(self,cube,update_database=True):self.cmd_add_cubes((cube,),update_database)defcmd_add_cubes(self,cubes,update_database=True):"""update_database is telling if the database schema should be updated or if only the relevant eproperty should be inserted (for the case where a cube has been extracted from an existing instance, so the cube schema is already in there) """newcubes=super(ServerMigrationHelper,self).cmd_add_cubes(cubes)ifnotnewcubes:returnforcubeinnewcubes:self.cmd_set_property('system.version.'+cube,self.config.cube_version(cube))ifcubeinSOURCE_TYPES:# don't use config.sources() in case some sources have been# disabled for migrationsourcescfg=self.config.read_sources_file()sourcescfg[cube]=ask_source_config(cube)self.config.write_sources_file(sourcescfg)clear_cache(self.config,'read_sources_file')ifnotupdate_database:self.commit()returnnewcubes_schema=self.config.load_schema(construction_mode='non-strict')new=set()# execute pre-create filesforpackinreversed(newcubes):self.exec_event_script('precreate',self.config.cube_dir(pack))# add new entity and relation typesforrschemainnewcubes_schema.relations():ifnotrschemainself.repo.schema:self.cmd_add_relation_type(rschema.type)new.add(rschema.type)toadd=[eschemaforeschemainnewcubes_schema.entities()ifnoteschemainself.repo.schema]foreschemainorder_eschemas(toadd):self.cmd_add_entity_type(eschema.type)new.add(eschema.type)# check if attributes has been added to existing entitiesforrschemainnewcubes_schema.relations():existingschema=self.repo.schema.rschema(rschema.type)for(fromtype,totype)inrschema.iter_rdefs():ifexistingschema.has_rdef(fromtype,totype):continue# check we should actually add the relation definitionifnot(fromtypeinnewortotypeinneworrschemainnew):continueself.cmd_add_relation_definition(str(fromtype),rschema.type,str(totype))# execute post-create filesforpackinreversed(newcubes):self.exec_event_script('postcreate',self.config.cube_dir(pack))self.commit()defcmd_remove_cube(self,cube,removedeps=False):removedcubes=super(ServerMigrationHelper,self).cmd_remove_cube(cube,removedeps)ifnotremovedcubes:returnfsschema=self.fs_schemaremovedcubes_schema=self.config.load_schema(construction_mode='non-strict')reposchema=self.repo.schema# execute pre-remove filesforpackinreversed(removedcubes):self.exec_event_script('preremove',self.config.cube_dir(pack))# remove cubes'entity and relation typesforrschemainfsschema.relations():ifnotrschemainremovedcubes_schemaandrschemainreposchema:self.cmd_drop_relation_type(rschema.type)toremove=[eschemaforeschemainfsschema.entities()ifnoteschemainremovedcubes_schemaandeschemainreposchema]foreschemainreversed(order_eschemas(toremove)):self.cmd_drop_entity_type(eschema.type)forrschemainfsschema.relations():ifrschemainremovedcubes_schemaandrschemainreposchema:# check if attributes/relations has been added to entities from# other cubesforfromtype,totypeinrschema.iter_rdefs():ifnotremovedcubes_schema[rschema.type].has_rdef(fromtype,totype)and \reposchema[rschema.type].has_rdef(fromtype,totype):self.cmd_drop_relation_definition(str(fromtype),rschema.type,str(totype))# execute post-remove filesforpackinreversed(removedcubes):self.exec_event_script('postremove',self.config.cube_dir(pack))self.rqlexec('DELETE CWProperty X WHERE X pkey %(pk)s',{'pk':u'system.version.'+pack},ask_confirm=False)self.commit()# schema migration actions ################################################defcmd_add_attribute(self,etype,attrname,attrtype=None,commit=True):"""add a new attribute on the given entity type"""ifattrtypeisNone:rschema=self.fs_schema.rschema(attrname)attrtype=rschema.objects(etype)[0]self.cmd_add_relation_definition(etype,attrname,attrtype,commit=commit)defcmd_drop_attribute(self,etype,attrname,commit=True):"""drop an existing attribute from the given entity type `attrname` is a string giving the name of the attribute to drop """rschema=self.repo.schema.rschema(attrname)attrtype=rschema.objects(etype)[0]self.cmd_drop_relation_definition(etype,attrname,attrtype,commit=commit)defcmd_rename_attribute(self,etype,oldname,newname,commit=True):"""rename an existing attribute of the given entity type `oldname` is a string giving the name of the existing attribute `newname` is a string giving the name of the renamed attribute """eschema=self.fs_schema.eschema(etype)attrtype=eschema.destination(newname)# have to commit this first step anyway to get the definition# actually in the schemaself.cmd_add_attribute(etype,newname,attrtype,commit=True)# skipp NULL values if the attribute is requiredrql='SET X %s VAL WHERE X is %s, X %s VAL'%(newname,etype,oldname)card=eschema.rproperty(newname,'cardinality')[0]ifcard=='1':rql+=', NOT X %s NULL'%oldnameself.rqlexec(rql,ask_confirm=self.verbosity>=2)self.cmd_drop_attribute(etype,oldname,commit=commit)defcmd_add_entity_type(self,etype,auto=True,commit=True):"""register a new entity type in auto mode, automatically register entity's relation where the targeted type is known """instschema=self.repo.schemaifetypeininstschema:# XXX (syt) plz explain: if we're adding an entity type, it should# not be there...eschema=instschema[etype]ifeschema.is_final():instschema.del_entity_type(etype)else:eschema=self.fs_schema.eschema(etype)confirm=self.verbosity>=2# register the entity into CWETypeself.rqlexecall(ss.eschema2rql(eschema),ask_confirm=confirm)# add specializes relation if neededself.rqlexecall(ss.eschemaspecialize2rql(eschema),ask_confirm=confirm)# register groups / permissions for the entityself.rqlexecall(ss.erperms2rql(eschema,self.group_mapping()),ask_confirm=confirm)# register entity's attributesforrschema,attrschemaineschema.attribute_definitions():# ignore those meta relations, they will be automatically addedifrschema.typeinMETA_RTYPES:continueifnotrschema.typeininstschema:# need to add the relation type and to commit to get it# actually in the schemaself.cmd_add_relation_type(rschema.type,False,commit=True)# register relation definitionself.rqlexecall(ss.rdef2rql(rschema,etype,attrschema.type),ask_confirm=confirm)# take care to newly introduced base class# XXX some part of this should probably be under the "if auto" blockforspschemaineschema.specialized_by(recursive=False):try:instspschema=instschema[spschema]exceptKeyError:# specialized entity type not in schema, ignorecontinueifinstspschema.specializes()!=eschema:self.rqlexec('SET D specializes P WHERE D eid %(d)s, P name %(pn)s',{'d':instspschema.eid,'pn':eschema.type},ask_confirm=confirm)forrschema,tschemas,roleinspschema.relation_definitions(True):fortschemaintschemas:ifnottschemaininstschema:continueifrole=='subject':subjschema=spschemaobjschema=tschemaifrschema.finalandinstspschema.has_subject_relation(rschema):# attribute already set, has_rdef would check if# it's of the same type, we don't want this so# simply skip herecontinueelifrole=='object':subjschema=tschemaobjschema=spschemaif(rschema.rproperty(subjschema,objschema,'infered')or(instschema.has_relation(rschema)andinstschema[rschema].has_rdef(subjschema,objschema))):continueself.cmd_add_relation_definition(subjschema.type,rschema.type,objschema.type)ifauto:# we have commit here to get relation types actually in the schemaself.commit()added=[]forrschemaineschema.subject_relations():# attribute relation have already been processed and# 'owned_by'/'created_by' will be automatically addedifrschema.finalorrschema.typeinMETA_RTYPES:continuertypeadded=rschema.typeininstschemafortargetschemainrschema.objects(etype):# ignore relations where the targeted type is not in the# current instance schematargettype=targetschema.typeifnottargettypeininstschemaandtargettype!=etype:continueifnotrtypeadded:# need to add the relation type and to commit to get it# actually in the schemaadded.append(rschema.type)self.cmd_add_relation_type(rschema.type,False,commit=True)rtypeadded=True# register relation definition# remember this two avoid adding twice non symetric relation# such as "Emailthread forked_from Emailthread"added.append((etype,rschema.type,targettype))self.rqlexecall(ss.rdef2rql(rschema,etype,targettype),ask_confirm=confirm)forrschemaineschema.object_relations():rtypeadded=rschema.typeininstschemaorrschema.typeinaddedfortargetschemainrschema.subjects(etype):# ignore relations where the targeted type is not in the# current instance schematargettype=targetschema.type# don't check targettype != etype since in this case the# relation has already been added as a subject relationifnottargettypeininstschema:continueifnotrtypeadded:# need to add the relation type and to commit to get it# actually in the schemaself.cmd_add_relation_type(rschema.type,False,commit=True)rtypeadded=Trueelif(targettype,rschema.type,etype)inadded:continue# register relation definitionself.rqlexecall(ss.rdef2rql(rschema,targettype,etype),ask_confirm=confirm)ifcommit:self.commit()defcmd_drop_entity_type(self,etype,commit=True):"""unregister an existing entity type This will trigger deletion of necessary relation types and definitions """# XXX what if we delete an entity type which is specialized by other types# unregister the entity from CWETypeself.rqlexec('DELETE CWEType X WHERE X name %(etype)s',{'etype':etype},ask_confirm=self.verbosity>=2)ifcommit:self.commit()defcmd_rename_entity_type(self,oldname,newname,commit=True):"""rename an existing entity type in the persistent schema `oldname` is a string giving the name of the existing entity type `newname` is a string giving the name of the renamed entity type """self.rqlexec('SET ET name %(newname)s WHERE ET is CWEType, ET name %(oldname)s',{'newname':unicode(newname),'oldname':oldname},ask_confirm=False)ifcommit:self.commit()defcmd_add_relation_type(self,rtype,addrdef=True,commit=True):"""register a new relation type named `rtype`, as described in the schema description file. `addrdef` is a boolean value; when True, it will also add all relations of the type just added found in the schema definition file. Note that it implies an intermediate "commit" which commits the relation type creation (but not the relation definitions themselves, for which committing depends on the `commit` argument value). """rschema=self.fs_schema.rschema(rtype)# register the relation into CWRType and insert necessary relation# definitionsself.rqlexecall(ss.rschema2rql(rschema,addrdef=False),ask_confirm=self.verbosity>=2)# register groups / permissions for the relationself.rqlexecall(ss.erperms2rql(rschema,self.group_mapping()),ask_confirm=self.verbosity>=2)ifaddrdef:self.commit()self.rqlexecall(ss.rdef2rql(rschema),ask_confirm=self.verbosity>=2)ifrtypeinMETA_RTYPES:# if the relation is in META_RTYPES, ensure we're adding it for# all entity types *in the persistent schema*, not only those in# the fs schemaforetypeinself.repo.schema.entities():ifnotetypeinself.fs_schema:# get sample object type and rpropertiesobjtypes=rschema.objects()assertlen(objtypes)==1objtype=objtypes[0]props=rschema.rproperties(rschema.subjects(objtype)[0],objtype)assertpropsself.rqlexecall(ss.rdef2rql(rschema,etype,objtype,props),ask_confirm=self.verbosity>=2)ifcommit:self.commit()defcmd_drop_relation_type(self,rtype,commit=True):"""unregister an existing relation type"""# unregister the relation from CWRTypeself.rqlexec('DELETE CWRType X WHERE X name %r'%rtype,ask_confirm=self.verbosity>=2)ifcommit:self.commit()defcmd_rename_relation(self,oldname,newname,commit=True):"""rename an existing relation `oldname` is a string giving the name of the existing relation `newname` is a string giving the name of the renamed relation """self.cmd_add_relation_type(newname,commit=True)self.rqlexec('SET X %s Y WHERE X %s Y'%(newname,oldname),ask_confirm=self.verbosity>=2)self.cmd_drop_relation_type(oldname,commit=commit)defcmd_add_relation_definition(self,subjtype,rtype,objtype,commit=True):"""register a new relation definition, from its definition found in the schema definition file """rschema=self.fs_schema.rschema(rtype)ifnotrtypeinself.repo.schema:self.cmd_add_relation_type(rtype,addrdef=False,commit=True)self.rqlexecall(ss.rdef2rql(rschema,subjtype,objtype),ask_confirm=self.verbosity>=2)ifcommit:self.commit()defcmd_drop_relation_definition(self,subjtype,rtype,objtype,commit=True):"""unregister an existing relation definition"""rschema=self.repo.schema.rschema(rtype)# unregister the definition from CWAttribute or CWRelationifrschema.is_final():etype='CWAttribute'else:etype='CWRelation'rql=('DELETE %s X WHERE X from_entity FE, FE name "%s",''X relation_type RT, RT name "%s", X to_entity TE, TE name "%s"')self.rqlexec(rql%(etype,subjtype,rtype,objtype),ask_confirm=self.verbosity>=2)ifcommit:self.commit()defcmd_sync_schema_props_perms(self,ertype=None,syncperms=True,syncprops=True,syncrdefs=True,commit=True):"""synchronize the persistent schema against the current definition schema. It will synch common stuff between the definition schema and the actual persistent schema, it won't add/remove any entity or relation. """assertsyncpermsorsyncprops,'nothing to do'ifertypeisnotNone:ifisinstance(ertype,(tuple,list)):assertlen(ertype)==3,'not a relation definition'assertsyncprops,'can\'t update permission for a relation definition'self._synchronize_rdef_schema(*ertype)elifsyncprops:erschema=self.repo.schema[ertype]ifisinstance(erschema,CubicWebRelationSchema):self._synchronize_rschema(erschema,syncperms=syncperms,syncrdefs=syncrdefs)else:self._synchronize_eschema(erschema,syncperms=syncperms)else:self._synchronize_permissions(ertype)else:foretypeinself.repo.schema.entities():ifsyncprops:self._synchronize_eschema(etype,syncperms=syncperms)else:self._synchronize_permissions(etype)ifcommit:self.commit()defcmd_change_relation_props(self,subjtype,rtype,objtype,commit=True,**kwargs):"""change some properties of a relation definition you usually want to use sync_schema_props_perms instead. """assertkwargsrestriction=[]ifsubjtypeandsubjtype!='Any':restriction.append('X from_entity FE, FE name "%s"'%subjtype)ifobjtypeandobjtype!='Any':restriction.append('X to_entity TE, TE name "%s"'%objtype)ifrtypeandrtype!='Any':restriction.append('X relation_type RT, RT name "%s"'%rtype)assertrestrictionvalues=[]fork,vinkwargs.items():values.append('X %s%%(%s)s'%(k,k))ifisinstance(v,str):kwargs[k]=unicode(v)rql='SET %s WHERE %s'%(','.join(values),','.join(restriction))self.rqlexec(rql,kwargs,ask_confirm=self.verbosity>=2)ifcommit:self.commit()defcmd_set_size_constraint(self,etype,rtype,size,commit=True):"""set change size constraint of a string attribute if size is None any size constraint will be removed. you usually want to use sync_schema_props_perms instead. """oldvalue=Noneforconstrinself.repo.schema.eschema(etype).constraints(rtype):ifisinstance(constr,SizeConstraint):oldvalue=constr.maxifoldvalue==size:returnifoldvalueisNoneandnotsizeisNone:ceid=self.rqlexec('INSERT CWConstraint C: C value %(v)s, C cstrtype CT ''WHERE CT name "SizeConstraint"',{'v':SizeConstraint(size).serialize()},ask_confirm=self.verbosity>=2)[0][0]self.rqlexec('SET X constrained_by C WHERE X from_entity S, X relation_type R, ''S name "%s", R name "%s", C eid %s'%(etype,rtype,ceid),ask_confirm=self.verbosity>=2)elifnotoldvalueisNone:ifnotsizeisNone:self.rqlexec('SET C value %%(v)s WHERE X from_entity S, X relation_type R,''X constrained_by C, C cstrtype CT, CT name "SizeConstraint",''S name "%s", R name "%s"'%(etype,rtype),{'v':unicode(SizeConstraint(size).serialize())},ask_confirm=self.verbosity>=2)else:self.rqlexec('DELETE X constrained_by C WHERE X from_entity S, X relation_type R,''X constrained_by C, C cstrtype CT, CT name "SizeConstraint",''S name "%s", R name "%s"'%(etype,rtype),ask_confirm=self.verbosity>=2)# cleanup unused constraintsself.rqlexec('DELETE CWConstraint C WHERE NOT X constrained_by C')ifcommit:self.commit()@deprecated('use sync_schema_props_perms(ertype, syncprops=False)')defcmd_synchronize_permissions(self,ertype,commit=True):self.cmd_sync_schema_props_perms(ertype,syncprops=False,commit=commit)# Workflows handling ######################################################defcmd_add_workflow(self,name,wfof,default=True,commit=False,**kwargs):self.session.set_pool()# ensure pool is setwf=self.cmd_create_entity('Workflow',name=unicode(name),**kwargs)ifnotisinstance(wfof,(list,tuple)):wfof=(wfof,)foretypeinwfof:rset=self.rqlexec('SET X workflow_of ET WHERE X eid %(x)s, ET name %(et)s',{'x':wf.eid,'et':etype},'x',ask_confirm=False)assertrset,'unexistant entity type %s'%etypeifdefault:self.rqlexec('SET ET default_workflow X WHERE X eid %(x)s, ET name %(et)s',{'x':wf.eid,'et':etype},'x',ask_confirm=False)ifcommit:self.commit()returnwf# XXX remove once cmd_add_[state|transition] are removeddef_get_or_create_wf(self,etypes):self.session.set_pool()# ensure pool is setifnotisinstance(etypes,(list,tuple)):etypes=(etypes,)rset=self.rqlexec('Workflow X WHERE X workflow_of ET, ET name %(et)s',{'et':etypes[0]})ifrset:returnrset.get_entity(0,0)returnself.cmd_add_workflow('%s workflow'%';'.join(etypes),etypes)@deprecated('use add_workflow and Workflow.add_state method')defcmd_add_state(self,name,stateof,initial=False,commit=False,**kwargs):"""method to ease workflow definition: add a state for one or more entity type(s) """wf=self._get_or_create_wf(stateof)state=wf.add_state(name,initial,**kwargs)ifcommit:self.commit()returnstate.eid@deprecated('use add_workflow and Workflow.add_transition method')defcmd_add_transition(self,name,transitionof,fromstates,tostate,requiredgroups=(),conditions=(),commit=False,**kwargs):"""method to ease workflow definition: add a transition for one or more entity type(s), from one or more state and to a single state """wf=self._get_or_create_wf(transitionof)tr=wf.add_transition(name,fromstates,tostate,requiredgroups,conditions,**kwargs)ifcommit:self.commit()returntr.eid@deprecated('use Transition.set_transition_permissions method')defcmd_set_transition_permissions(self,treid,requiredgroups=(),conditions=(),reset=True,commit=False):"""set or add (if `reset` is False) groups and conditions for a transition """self.session.set_pool()# ensure pool is settr=self.session.entity_from_eid(treid)tr.set_transition_permissions(requiredgroups,conditions,reset)ifcommit:self.commit()@deprecated('use entity.fire_transition("transition") or entity.change_state("state")')defcmd_set_state(self,eid,statename,commit=False):self.session.set_pool()# ensure pool is setself.session.entity_from_eid(eid).change_state(statename)ifcommit:self.commit()# CWProperty handling ######################################################defcmd_property_value(self,pkey):rql='Any V WHERE X is CWProperty, X pkey %(k)s, X value V'rset=self.rqlexec(rql,{'k':pkey},ask_confirm=False)returnrset[0][0]defcmd_set_property(self,pkey,value):value=unicode(value)try:prop=self.rqlexec('CWProperty X WHERE X pkey %(k)s',{'k':pkey},ask_confirm=False).get_entity(0,0)except:self.cmd_create_entity('CWProperty',pkey=unicode(pkey),value=value)else:self.rqlexec('SET X value %(v)s WHERE X pkey %(k)s',{'k':pkey,'v':value},ask_confirm=False)# other data migration commands ###########################################defcmd_create_entity(self,etype,*args,**kwargs):"""add a new entity of the given type"""commit=kwargs.pop('commit',False)self.session.set_pool()entity=self.session.create_entity(etype,*args,**kwargs)ifcommit:self.commit()returnentity@deprecated('use create_entity')defcmd_add_entity(self,etype,*args,**kwargs):"""add a new entity of the given type"""returnself.cmd_create_entity(etype,*args,**kwargs).eiddefsqlexec(self,sql,args=None,ask_confirm=True):"""execute the given sql if confirmed should only be used for low level stuff undoable with existing higher level actions """ifnotask_confirmorself.confirm('execute sql: %s ?'%sql):self.session.set_pool()# ensure pool is settry:cu=self.session.system_sql(sql,args)except:ex=sys.exc_info()[1]ifself.confirm('error: %s\nabort?'%ex):raisereturntry:returncu.fetchall()except:# no result to fetchreturndefrqlexec(self,rql,kwargs=None,cachekey=None,ask_confirm=True):"""rql action"""ifnotisinstance(rql,(tuple,list)):rql=((rql,kwargs),)res=Noneself.session.set_pool()forrql,kwargsinrql:ifkwargs:msg='%s (%s)'%(rql,kwargs)else:msg=rqlifnotask_confirmorself.confirm('execute rql: %s ?'%msg):try:res=self.session.execute(rql,kwargs,cachekey)exceptException,ex:ifself.confirm('error: %s\nabort?'%ex):raisereturnresdefrqliter(self,rql,kwargs=None,ask_confirm=True):returnForRqlIterator(self,rql,None,ask_confirm)defcmd_deactivate_verification_hooks(self):self.repo.hm.deactivate_verification_hooks()defcmd_reactivate_verification_hooks(self):self.repo.hm.reactivate_verification_hooks()# broken db commands ######################################################defcmd_change_attribute_type(self,etype,attr,newtype,commit=True):"""low level method to change the type of an entity attribute. This is a quick hack which has some drawback: * only works when the old type can be changed to the new type by the underlying rdbms (eg using ALTER TABLE) * the actual schema won't be updated until next startup """rschema=self.repo.schema.rschema(attr)oldtype=rschema.objects(etype)[0]rdefeid=rschema.rproperty(etype,oldtype,'eid')sql=("UPDATE CWAttribute ""SET to_entity=(SELECT eid FROM CWEType WHERE name='%s')""WHERE eid=%s")%(newtype,rdefeid)self.sqlexec(sql,ask_confirm=False)dbhelper=self.repo.system_source.dbhelpersqltype=dbhelper.TYPE_MAPPING[newtype]sql='ALTER TABLE %s ALTER COLUMN %s TYPE %s'%(etype,attr,sqltype)self.sqlexec(sql,ask_confirm=False)ifcommit:self.commit()defcmd_add_entity_type_table(self,etype,commit=True):"""low level method to create the sql table for an existing entity. This may be useful on accidental desync between the repository schema and a sql database """dbhelper=self.repo.system_source.dbhelpertablesql=eschema2sql(dbhelper,self.repo.schema.eschema(etype),prefix=SQL_PREFIX)forsqlintablesql.split(';'):ifsql.strip():self.sqlexec(sql)ifcommit:self.commit()defcmd_add_relation_type_table(self,rtype,commit=True):"""low level method to create the sql table for an existing relation. This may be useful on accidental desync between the repository schema and a sql database """dbhelper=self.repo.system_source.dbhelpertablesql=rschema2sql(dbhelper,self.repo.schema.rschema(rtype))forsqlintablesql.split(';'):ifsql.strip():self.sqlexec(sql)ifcommit:self.commit()classForRqlIterator:"""specific rql iterator to make the loop skipable"""def__init__(self,helper,rql,kwargs,ask_confirm):self._h=helperself.rql=rqlself.kwargs=kwargsself.ask_confirm=ask_confirmself._rsetit=Nonedef__iter__(self):returnselfdefnext(self):ifself._rsetitisnotNone:returnself._rsetit.next()rql,kwargs=self.rql,self.kwargsifkwargs:msg='%s (%s)'%(rql,kwargs)else:msg=rqlifself.ask_confirm:ifnotself._h.confirm('execute rql: %s ?'%msg):raiseStopIterationself._h.session.set_pool()try:rset=self._h.session.execute(rql,kwargs)exceptException,ex:ifself._h.confirm('error: %s\nabort?'%ex):raiseelse:raiseStopIterationself._rsetit=iter(rset)returnself._rsetit.next()