[repo, entity] move entity cache initialization to a function, and call it before source.add_entity so it may be used in error handler or such
# copyright 2003-2010 LOGILAB S.A. (Paris, FRANCE), all rights reserved.# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr## This file is part of CubicWeb.## CubicWeb is free software: you can redistribute it and/or modify it under the# terms of the GNU Lesser General Public License as published by the Free# Software Foundation, either version 2.1 of the License, or (at your option)# any later version.## CubicWeb is distributed in the hope that it will be useful, but WITHOUT# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more# details.## You should have received a copy of the GNU Lesser General Public License along# with CubicWeb. If not, see <http://www.gnu.org/licenses/>."""functions for schema / permissions (de)serialization using RQL"""__docformat__="restructuredtext en"importosfromitertoolsimportchainfromlogilab.common.shellutilsimportProgressBarfromyamsimportBadSchemaDefinition,schemaasschemamod,buildobjsasybofromcubicwebimportCW_SOFTWARE_ROOT,typed_eidfromcubicweb.schemaimport(CONSTRAINTS,ETYPE_NAME_MAP,VIRTUAL_RTYPES,PURE_VIRTUAL_RTYPES)fromcubicweb.serverimportsqlutilsdefgroup_mapping(cursor,interactive=True):"""create a group mapping from an rql cursor A group mapping has standard group names as key (managers, owners at least) and the actual CWGroup entity's eid as associated value. In interactive mode (the default), missing groups'eid will be prompted from the user. """res={}foreid,nameincursor.execute('Any G, N WHERE G is CWGroup, G name N',build_descr=False):res[name]=eidifnotinteractive:returnresmissing=[gforgin('owners','managers','users','guests')ifnotginres]ifmissing:print'some native groups are missing but the following groups have been found:'print'\n'.join('* %s (%s)'%(n,eid)forn,eidinres.items())printprint'enter the eid of a to group to map to each missing native group'print'or just type enter to skip permissions granted to a group'forgroupinmissing:whileTrue:value=raw_input('eid for group %s: '%group).strip()ifnotvalue:continuetry:eid=typed_eid(value)exceptValueError:print'eid should be an integer'continueforeid_inres.values():ifeid==eid_:breakelse:print'eid is not a group eid'continueres[name]=eidbreakreturnresdefcstrtype_mapping(cursor):"""cached constraint types mapping"""map=dict(cursor.execute('Any T, X WHERE X is CWConstraintType, X name T'))ifnot'BoundConstraint'inmap:map['BoundConstraint']=map['BoundaryConstraint']returnmap# schema / perms deserialization ##############################################defdeserialize_schema(schema,session):"""return a schema according to information stored in an rql database as CWRType and CWEType entities """repo=session.repodbhelper=repo.system_source.dbhelper# XXX bw compat (3.6 migration)sqlcu=session.pool['system']sqlcu.execute("SELECT * FROM cw_CWRType WHERE cw_name='symetric'")ifsqlcu.fetchall():sql=dbhelper.sql_rename_col('cw_CWRType','cw_symetric','cw_symmetric',dbhelper.TYPE_MAPPING['Boolean'],True)sqlcu.execute(sql)sqlcu.execute("UPDATE cw_CWRType SET cw_name='symmetric' WHERE cw_name='symetric'")session.commit(False)ertidx={}copiedeids=set()permsidx=deserialize_ertype_permissions(session)schema.reading_from_database=Trueforeid,etype,descinsession.execute('Any X, N, D WHERE X is CWEType, X name N, X description D',build_descr=False):# base types are already in the schema, skip themifetypeinschemamod.BASE_TYPES:# just set the eideschema=schema.eschema(etype)eschema.eid=eidertidx[eid]=etypecontinueifetypeinETYPE_NAME_MAP:needcopy=Falsenetype=ETYPE_NAME_MAP[etype]# can't use write rql queries at this point, use raw sqlsqlexec=session.system_sqlifsqlexec('SELECT 1 FROM %(p)sCWEType WHERE %(p)sname=%%(n)s'%{'p':sqlutils.SQL_PREFIX},{'n':netype}).fetchone():# the new type already exists, we should copy (eg make existing# instances of the old type instances of the new type)assertetype.lower()!=netype.lower()needcopy=Trueelse:# the new type doesn't exist, we should renamesqlexec('UPDATE %(p)sCWEType SET %(p)sname=%%(n)s WHERE %(p)seid=%%(x)s'%{'p':sqlutils.SQL_PREFIX},{'x':eid,'n':netype})ifetype.lower()!=netype.lower():sqlexec('ALTER TABLE %s%s RENAME TO %s%s'%(sqlutils.SQL_PREFIX,etype,sqlutils.SQL_PREFIX,netype))sqlexec('UPDATE entities SET type=%(n)s WHERE type=%(x)s',{'x':etype,'n':netype})session.commit(False)try:sqlexec('UPDATE deleted_entities SET type=%(n)s WHERE type=%(x)s',{'x':etype,'n':netype})except:passtocleanup=[eid]tocleanup+=(eidforeid,(eidetype,uri,extid)inrepo._type_source_cache.items()ifetype==eidetype)repo.clear_caches(tocleanup)session.commit(False)ifneedcopy:ertidx[eid]=netypecopiedeids.add(eid)# copy / CWEType entity removal expected to be done through# rename_entity_type in a migration scriptcontinueetype=netypeertidx[eid]=etypeeschema=schema.add_entity_type(ybo.EntityType(name=etype,description=desc,eid=eid))set_perms(eschema,permsidx)foretype,stypeinsession.execute('Any XN, ETN WHERE X is CWEType, X name XN, X specializes ET, ET name ETN',build_descr=False):etype=ETYPE_NAME_MAP.get(etype,etype)stype=ETYPE_NAME_MAP.get(stype,stype)schema.eschema(etype)._specialized_type=stypeschema.eschema(stype)._specialized_by.append(etype)foreid,rtype,desc,sym,il,ftcinsession.execute('Any X,N,D,S,I,FTC WHERE X is CWRType, X name N, X description D, ''X symmetric S, X inlined I, X fulltext_container FTC',build_descr=False):ertidx[eid]=rtyperschema=schema.add_relation_type(ybo.RelationType(name=rtype,description=desc,symmetric=bool(sym),inlined=bool(il),fulltext_container=ftc,eid=eid))cstrsidx=deserialize_rdef_constraints(session)pendingrdefs=[]# closure to factorize common code of attribute/relation rdef additiondef_add_rdef(rdefeid,seid,reid,oeid,**kwargs):rdef=ybo.RelationDefinition(ertidx[seid],ertidx[reid],ertidx[oeid],constraints=cstrsidx.get(rdefeid,()),eid=rdefeid,**kwargs)ifseidincopiedeidsoroeidincopiedeids:# delay addition of this rdef. We'll insert them later if needed. We# have to do this because:## * on etype renaming, we want relation of the old entity type being# redirected to the new type during migration## * in the case of a copy, we've to take care that rdef already# existing in the schema are not overwritten by a redirected one,# since we want correct eid on them (redirected rdef will be# removed in rename_entity_type)pendingrdefs.append(rdef)else:# add_relation_def return a RelationDefinitionSchema if it has been# actually added (can be None on duplicated relation definitions,# e.g. if the relation type is marked as beeing symmetric)rdefs=schema.add_relation_def(rdef)ifrdefsisnotNone:ertidx[rdefeid]=rdefsset_perms(rdefs,permsidx)forvaluesinsession.execute('Any X,SE,RT,OE,CARD,ORD,DESC,IDX,FTIDX,I18N,DFLT WHERE X is CWAttribute,''X relation_type RT, X cardinality CARD, X ordernum ORD, X indexed IDX,''X description DESC, X internationalizable I18N, X defaultval DFLT,''X fulltextindexed FTIDX, X from_entity SE, X to_entity OE',build_descr=False):rdefeid,seid,reid,oeid,card,ord,desc,idx,ftidx,i18n,default=values_add_rdef(rdefeid,seid,reid,oeid,cardinality=card,description=desc,order=ord,indexed=idx,fulltextindexed=ftidx,internationalizable=i18n,default=default)forvaluesinsession.execute('Any X,SE,RT,OE,CARD,ORD,DESC,C WHERE X is CWRelation, X relation_type RT,''X cardinality CARD, X ordernum ORD, X description DESC, ''X from_entity SE, X to_entity OE, X composite C',build_descr=False):rdefeid,seid,reid,oeid,card,ord,desc,comp=values_add_rdef(rdefeid,seid,reid,oeid,cardinality=card,description=desc,order=ord,composite=comp)forrdefinpendingrdefs:try:rdefs=schema.add_relation_def(rdef)exceptBadSchemaDefinition:continueifrdefsisnotNone:set_perms(rdefs,permsidx)unique_togethers={}try:rset=session.execute('Any X,E,R WHERE ''X is CWUniqueTogetherConstraint, ''X constraint_of E, X relations R',build_descr=False)exceptException:session.rollback()# first migration introducing CWUniqueTogetherConstraint cw 3.9.6else:forvaluesinrset:uniquecstreid,eeid,releid=valueseschema=schema.schema_by_eid(eeid)relations=unique_togethers.setdefault(uniquecstreid,(eschema,[]))relations[1].append(ertidx[releid].rtype.type)foreschema,unique_togetherinunique_togethers.itervalues():eschema._unique_together.append(tuple(sorted(unique_together)))schema.infer_specialization_rules()session.commit()schema.reading_from_database=Falsedefdeserialize_ertype_permissions(session):"""return sect action:groups associations for the given entity or relation schema with its eid, according to schema's permissions stored in the database as [read|add|delete|update]_permission relations between CWEType/CWRType and CWGroup entities """res={}foractionin('read','add','update','delete'):rql='Any E,N WHERE G is CWGroup, G name N, E %s_permission G'%actionforeid,gnameinsession.execute(rql,build_descr=False):res.setdefault(eid,{}).setdefault(action,[]).append(gname)rql=('Any E,X,EXPR,V WHERE X is RQLExpression, X expression EXPR, ''E %s_permission X, X mainvars V'%action)foreid,expreid,expr,mainvarsinsession.execute(rql,build_descr=False):# we don't know yet if it's a rql expr for an entity or a relation,# so append a tuple to differentiate from groups and so we'll be# able to instantiate it laterres.setdefault(eid,{}).setdefault(action,[]).append((expr,mainvars,expreid))returnresdefdeserialize_rdef_constraints(session):"""return the list of relation definition's constraints as instances"""res={}forrdefeid,ceid,ct,valinsession.execute('Any E, X,TN,V WHERE E constrained_by X, X is CWConstraint, ''X cstrtype T, T name TN, X value V',build_descr=False):cstr=CONSTRAINTS[ct].deserialize(val)cstr.eid=ceidres.setdefault(rdefeid,[]).append(cstr)returnresdefset_perms(erschema,permsidx):"""set permissions on the given erschema according to the permission definition dictionary as built by deserialize_ertype_permissions for a given erschema's eid """# reset erschema permissions here to avoid getting yams default anywayerschema.permissions=dict((action,())foractioninerschema.ACTIONS)try:thispermsdict=permsidx[erschema.eid]exceptKeyError:returnforaction,somethingsinthispermsdict.iteritems():# XXX cw < 3.6.1 bw compatifisinstance(erschema,schemamod.RelationDefinitionSchema)anderschema.finalandaction=='add':action='update'erschema.permissions[action]=tuple(isinstance(p,tuple)anderschema.rql_expression(*p)orpforpinsomethings)# schema / perms serialization ################################################defserialize_schema(cursor,schema):"""synchronize schema and permissions in the database according to current schema """quiet=os.environ.get('APYCOT_ROOT')ifnotquiet:_title='-> storing the schema in the database 'print_title,execute=cursor.executeeschemas=schema.entities()ifnotquiet:pb_size=(len(eschemas+schema.relations())+len(CONSTRAINTS)+len([xforxineschemasifx.specializes()]))pb=ProgressBar(pb_size,title=_title)else:pb=Nonegroupmap=group_mapping(cursor,interactive=False)# serialize all entity types, assuring CWEType is serialized first for proper# is / is_instance_of insertioneschemas.remove(schema.eschema('CWEType'))eschemas.insert(0,schema.eschema('CWEType'))foreschemaineschemas:execschemarql(execute,eschema,eschema2rql(eschema,groupmap))ifpbisnotNone:pb.update()# serialize constraint typescstrtypemap={}rql='INSERT CWConstraintType X: X name %(ct)s'forcstrtypeinCONSTRAINTS:ifcstrtype=='BoundConstraint':continue# XXX deprecated in yams 0.29 / cw 3.8.1cstrtypemap[cstrtype]=execute(rql,{'ct':unicode(cstrtype)},build_descr=False)[0][0]ifpbisnotNone:pb.update()cstrtypemap['BoundConstraint']=cstrtypemap['BoundaryConstraint']# serialize relationsforrschemainschema.relations():# skip virtual relations such as eid, has_text and identityifrschemainVIRTUAL_RTYPES:ifpbisnotNone:pb.update()continueexecschemarql(execute,rschema,rschema2rql(rschema,addrdef=False))ifrschema.symmetric:rdefs=[rdeffork,rdefinrschema.rdefs.iteritems()if(rdef.subject,rdef.object)==k]else:rdefs=rschema.rdefs.itervalues()forrdefinrdefs:execschemarql(execute,rdef,rdef2rql(rdef,cstrtypemap,groupmap))ifpbisnotNone:pb.update()# serialize unique_together constraintsforeschemaineschemas:forunique_togetherineschema._unique_together:execschemarql(execute,eschema,[uniquetogether2rql(eschema,unique_together)])forrql,kwargsinspecialize2rql(schema):execute(rql,kwargs,build_descr=False)ifpbisnotNone:pb.update()ifnotquiet:print# high level serialization functionsdefexecschemarql(execute,schema,rqls):forrql,kwargsinrqls:kwargs['x']=schema.eidrset=execute(rql,kwargs,build_descr=False)ifschema.eidisNone:schema.eid=rset[0][0]else:assertrsetdeferschema2rql(erschema,groupmap):ifisinstance(erschema,schemamod.EntitySchema):returneschema2rql(erschema,groupmap=groupmap)returnrschema2rql(erschema,groupmap=groupmap)defspecialize2rql(schema):foreschemainschema.entities():ifeschema.final:continueforrql,kwargsineschemaspecialize2rql(eschema):yieldrql,kwargs# etype serializationdefeschema2rql(eschema,groupmap=None):"""return a list of rql insert statements to enter an entity schema in the database as an CWEType entity """relations,values=eschema_relations_values(eschema)# NOTE: 'specializes' relation can't be inserted here since there's no# way to make sure the parent type is inserted before the child typeyield'INSERT CWEType X: %s'%','.join(relations),values# entity permissionsifgroupmapisnotNone:forrql,argsin_erperms2rql(eschema,groupmap):yieldrql,argsdefeschema_relations_values(eschema):values=_ervalues(eschema)relations=['X %s%%(%s)s'%(attr,attr)forattrinsorted(values)]returnrelations,valuesdefeschemaspecialize2rql(eschema):specialized_type=eschema.specializes()ifspecialized_type:values={'x':eschema.eid,'et':specialized_type.eid}yield'SET X specializes ET WHERE X eid %(x)s, ET eid %(et)s',valuesdefuniquetogether2rql(eschema,unique_together):relations=[]restrictions=[]substs={}fori,nameinenumerate(unique_together):rschema=eschema.rdef(name)var='R%d'%irtype='T%d'%isubsts[rtype]=rschema.rtype.typerelations.append('C relations %s'%var)restrictions.append('%(var)s from_entity X, ''%(var)s relation_type %(rtype)s, ''%(rtype)s name %%(%(rtype)s)s' \%{'var':var,'rtype':rtype})relations=', '.join(relations)restrictions=', '.join(restrictions)rql=('INSERT CWUniqueTogetherConstraint C: '' C constraint_of X, %s ''WHERE '' X eid %%(x)s, %s')returnrql%(relations,restrictions),substsdef_ervalues(erschema):try:type_=unicode(erschema.type)exceptUnicodeDecodeError,e:raiseException("can't decode %s [was %s]"%(erschema.type,e))try:desc=unicode(erschema.description)oru''exceptUnicodeDecodeError,e:raiseException("can't decode %s [was %s]"%(erschema.description,e))return{'name':type_,'final':erschema.final,'description':desc,}# rtype serializationdefrschema2rql(rschema,cstrtypemap=None,addrdef=True,groupmap=None):"""return a list of rql insert statements to enter a relation schema in the database as an CWRType entity """ifrschema.type=='has_text':returnrelations,values=rschema_relations_values(rschema)yield'INSERT CWRType X: %s'%','.join(relations),valuesifaddrdef:assertcstrtypemap# sort for testing purposeforrdefinsorted(rschema.rdefs.itervalues(),key=lambdax:(x.subject,x.object)):forrql,valuesinrdef2rql(rdef,cstrtypemap,groupmap):yieldrql,valuesdefrschema_relations_values(rschema):values=_ervalues(rschema)values['final']=rschema.finalvalues['symmetric']=rschema.symmetricvalues['inlined']=rschema.inlinedifisinstance(rschema.fulltext_container,str):values['fulltext_container']=unicode(rschema.fulltext_container)else:values['fulltext_container']=rschema.fulltext_containerrelations=['X %s%%(%s)s'%(attr,attr)forattrinsorted(values)]returnrelations,values# rdef serializationdefrdef2rql(rdef,cstrtypemap,groupmap=None):# don't serialize infered relationsifrdef.infered:returnrelations,values=_rdef_values(rdef)relations.append('X relation_type ER,X from_entity SE,X to_entity OE')values.update({'se':rdef.subject.eid,'rt':rdef.rtype.eid,'oe':rdef.object.eid})ifrdef.final:etype='CWAttribute'else:etype='CWRelation'yield'INSERT %s X: %s WHERE SE eid %%(se)s,ER eid %%(rt)s,OE eid %%(oe)s'%(etype,','.join(relations),),valuesforrql,valuesinconstraints2rql(cstrtypemap,rdef.constraints):yieldrql,values# no groupmap means "no security insertion"ifgroupmap:forrql,argsin_erperms2rql(rdef,groupmap):yieldrql,argsdef_rdef_values(rdef):amap={'order':'ordernum','default':'defaultval'}values={}forprop,defaultinrdef.rproperty_defs(rdef.object).iteritems():ifpropin('eid','constraints','uid','infered','permissions'):continuevalue=getattr(rdef,prop)# XXX type cast really necessary?ifpropin('indexed','fulltextindexed','internationalizable'):value=bool(value)elifprop=='ordernum':value=int(value)elifisinstance(value,str):value=unicode(value)ifvalueisnotNoneandprop=='default':ifvalueisFalse:value=u''ifnotisinstance(value,unicode):value=unicode(value)values[amap.get(prop,prop)]=valuerelations=['X %s%%(%s)s'%(attr,attr)forattrinsorted(values)]returnrelations,valuesdefconstraints2rql(cstrtypemap,constraints,rdefeid=None):forconstraintinconstraints:values={'ct':cstrtypemap[constraint.type()],'value':unicode(constraint.serialize()),'x':rdefeid}# when not specified, will have to be set by the calleryield'INSERT CWConstraint X: X value %(value)s, X cstrtype CT, EDEF constrained_by X WHERE \CT eid %(ct)s, EDEF eid %(x)s',valuesdef_erperms2rql(erschema,groupmap):"""return rql insert statements to enter the entity or relation schema's permissions in the database as [read|add|delete|update]_permission relations between CWEType/CWRType and CWGroup entities """foractioninerschema.ACTIONS:try:grantedto=erschema.action_permissions(action)exceptKeyError:# may occurs when modifying persistent schemacontinueforgroup_or_rqlexpringrantedto:ifisinstance(group_or_rqlexpr,basestring):# grouptry:yield('SET X %s_permission Y WHERE Y eid %%(g)s, X eid %%(x)s'%action,{'g':groupmap[group_or_rqlexpr]})exceptKeyError:continueelse:# rqlexprrqlexpr=group_or_rqlexpryield('INSERT RQLExpression E: E expression %%(e)s, E exprtype %%(t)s, ''E mainvars %%(v)s, X %s_permission E WHERE X eid %%(x)s'%action,{'e':unicode(rqlexpr.expression),'v':unicode(rqlexpr.mainvars),'t':unicode(rqlexpr.__class__.__name__)})# update functionsdefupdateeschema2rql(eschema,eid):relations,values=eschema_relations_values(eschema)values['x']=eidyield'SET %s WHERE X eid %%(x)s'%','.join(relations),valuesdefupdaterschema2rql(rschema,eid):relations,values=rschema_relations_values(rschema)values['x']=eidyield'SET %s WHERE X eid %%(x)s'%','.join(relations),valuesdefupdaterdef2rql(rdef,eid):relations,values=_rdef_values(rdef)values['x']=eidyield'SET %s WHERE X eid %%(x)s'%','.join(relations),values