Updated CW tutorial.
* Summarized the list of steps to create a new cube.
* Added a subsection on writing entities.
* Added a subsection on modifying the schema and updating the corresponding instance.
"""functions for schema / permissions (de)serialization using RQL:organization: Logilab:copyright: 2001-2010 LOGILAB S.A. (Paris, FRANCE), license is LGPL v2.:contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr:license: GNU Lesser General Public License, v2.1 - http://www.gnu.org/licenses"""__docformat__="restructuredtext en"importosimportsysimportosfromitertoolsimportchainfromlogilab.common.shellutilsimportProgressBarfromyamsimportschemaasschemamod,buildobjsasybofromcubicweb.schemaimportCONSTRAINTS,ETYPE_NAME_MAP,VIRTUAL_RTYPESfromcubicweb.serverimportsqlutilsdefgroup_mapping(cursor,interactive=True):"""create a group mapping from an rql cursor A group mapping has standard group names as key (managers, owners at least) and the actual CWGroup entity's eid as associated value. In interactive mode (the default), missing groups'eid will be prompted from the user. """res={}foreid,nameincursor.execute('Any G, N WHERE G is CWGroup, G name N',build_descr=False):res[name]=eidifnotinteractive:returnresmissing=[gforgin('owners','managers','users','guests')ifnotginres]ifmissing:print'some native groups are missing but the following groups have been found:'print'\n'.join('* %s (%s)'%(n,eid)forn,eidinres.items())printprint'enter the eid of a to group to map to each missing native group'print'or just type enter to skip permissions granted to a group'forgroupinmissing:whileTrue:value=raw_input('eid for group %s: '%group).strip()ifnotvalue:continuetry:res[group]=int(value)exceptValueError:print'eid should be an integer'continuereturnresdef_set_sql_prefix(prefix):"""3.2.0 migration function: allow to unset/reset SQL_PREFIX"""formodulein('checkintegrity','migractions','schemahooks','sources.rql2sql','sources.native','sqlutils'):try:sys.modules['cubicweb.server.%s'%module].SQL_PREFIX=prefixprint'changed SQL_PREFIX for %s'%moduleexceptKeyError:passdef_update_database(schema,sqlcu):"""3.2.0 migration function: update database schema by adding SQL_PREFIX to entity type tables and columns """foretypeinschema.entities():ifetype.final:continuetry:sql='ALTER TABLE %s RENAME TO cw_%s'%(etype,ETYPE_NAME_MAP.get(etype,etype))printsqlsqlcu.execute(sql)except:passforrschemainetype.subject_relations():ifrschema=='has_text':continueifrschema.finalorrschema.inlined:sql='ALTER TABLE cw_%s RENAME %s TO cw_%s'%(etype,rschema,rschema)printsqlsqlcu.execute(sql)# schema / perms deserialization ##############################################OLD_SCHEMA_TYPES=frozenset(('EFRDef','ENFRDef','ERType','EEType','EConstraintType','EConstraint','EGroup','EUser','ECache','EPermission','EProperty'))defdeserialize_schema(schema,session):"""return a schema according to information stored in an rql database as CWRType and CWEType entities """#repo=session.reposqlcu=session.pool['system']_3_2_migration=Falsedbhelper=repo.system_source.dbhelpertables=set(t.lower()fortindbhelper.list_tables(sqlcu))if'eetype'intables:_3_2_migration=True# 3.2 migration_set_sql_prefix('')# first rename entity types whose name changed in 3.2 without adding the# cw_ prefixforetypeinOLD_SCHEMA_TYPES:ifetype.lower()intables:sql='ALTER TABLE %s RENAME TO %s'%(etype,ETYPE_NAME_MAP[etype])printsqlsqlcu.execute(sql)# other table renaming done once schema has been readsidx={}permsdict=deserialize_ertype_permissions(session)schema.reading_from_database=Trueforeid,etype,descinsession.execute('Any X, N, D WHERE X is CWEType, X name N, X description D',build_descr=False):# base types are already in the schema, skip themifetypeinschemamod.BASE_TYPES:# just set the eideschema=schema.eschema(etype)eschema.eid=eidsidx[eid]=eschemacontinueifetypeinETYPE_NAME_MAP:netype=ETYPE_NAME_MAP[etype]# can't use write rql queries at this point, use raw sqlsession.system_sql('UPDATE %(p)sCWEType SET %(p)sname=%%(n)s WHERE %(p)seid=%%(x)s'%{'p':sqlutils.SQL_PREFIX},{'x':eid,'n':netype})session.system_sql('UPDATE entities SET type=%(n)s WHERE type=%(x)s',{'x':etype,'n':netype})# XXX should be donne as well on sqlite based sourcesifnotetypeinOLD_SCHEMA_TYPESand \(getattr(dbhelper,'case_sensitive',False)oretype.lower()!=netype.lower()):session.system_sql('ALTER TABLE %s%s RENAME TO %s%s'%(sqlutils.SQL_PREFIX,etype,sqlutils.SQL_PREFIX,netype))session.commit(False)try:session.system_sql('UPDATE deleted_entities SET type=%(n)s WHERE type=%(x)s',{'x':etype,'n':netype})except:passtocleanup=[eid]tocleanup+=(eidforeid,(eidetype,uri,extid)inrepo._type_source_cache.items()ifetype==eidetype)repo.clear_caches(tocleanup)session.commit(False)etype=netypeetype=ybo.EntityType(name=etype,description=desc,eid=eid)eschema=schema.add_entity_type(etype)sidx[eid]=eschemaset_perms(eschema,permsdict)foretype,stypeinsession.execute('Any XN, ETN WHERE X is CWEType, X name XN, X specializes ET, ET name ETN',build_descr=False):schema.eschema(etype)._specialized_type=stypeschema.eschema(stype)._specialized_by.append(etype)foreid,rtype,desc,sym,il,ftcinsession.execute('Any X,N,D,S,I,FTC WHERE X is CWRType, X name N, X description D, ''X symetric S, X inlined I, X fulltext_container FTC',build_descr=False):rtype=ybo.RelationType(name=rtype,description=desc,symetric=bool(sym),inlined=bool(il),fulltext_container=ftc,eid=eid)rschema=schema.add_relation_type(rtype)sidx[eid]=rschemacstrsdict=deserialize_rdef_constraints(session)forvaluesinsession.execute('Any X,SE,RT,OE,CARD,ORD,DESC,IDX,FTIDX,I18N,DFLT WHERE X is CWAttribute,''X relation_type RT, X cardinality CARD, X ordernum ORD, X indexed IDX,''X description DESC, X internationalizable I18N, X defaultval DFLT,''X fulltextindexed FTIDX, X from_entity SE, X to_entity OE',build_descr=False):rdefeid,seid,reid,teid,card,ord,desc,idx,ftidx,i18n,default=valuesrdef=ybo.RelationDefinition(sidx[seid].type,sidx[reid].type,sidx[teid].type,cardinality=card,constraints=cstrsdict.get(rdefeid,()),order=ord,description=desc,indexed=idx,fulltextindexed=ftidx,internationalizable=i18n,default=default,eid=rdefeid)rdefs=schema.add_relation_def(rdef)# rdefs can be None on duplicated relation definitions (e.g. symetrics)ifrdefsisnotNone:set_perms(rdefs,permsdict)forvaluesinsession.execute('Any X,SE,RT,OE,CARD,ORD,DESC,C WHERE X is CWRelation, X relation_type RT,''X cardinality CARD, X ordernum ORD, X description DESC, ''X from_entity SE, X to_entity OE, X composite C',build_descr=False):rdefeid,seid,reid,teid,card,ord,desc,c=valuesrdef=ybo.RelationDefinition(sidx[seid].type,sidx[reid].type,sidx[teid].type,constraints=cstrsdict.get(rdefeid,()),cardinality=card,order=ord,description=desc,composite=c,eid=rdefeid)rdefs=schema.add_relation_def(rdef)# rdefs can be None on duplicated relation definitions (e.g. symetrics)ifrdefsisnotNone:set_perms(rdefs,permsdict)schema.infer_specialization_rules()if_3_2_migration:_update_database(schema,sqlcu)_set_sql_prefix('cw_')session.commit()schema.reading_from_database=Falsedefdeserialize_ertype_permissions(session):"""return sect action:groups associations for the given entity or relation schema with its eid, according to schema's permissions stored in the database as [read|add|delete|update]_permission relations between CWEType/CWRType and CWGroup entities """res={}foractionin('read','add','update','delete'):rql='Any E,N WHERE G is CWGroup, G name N, E %s_permission G'%actionforeid,gnameinsession.execute(rql,build_descr=False):res.setdefault(eid,{}).setdefault(action,[]).append(gname)rql=('Any E,X,EXPR,V WHERE X is RQLExpression, X expression EXPR, ''E %s_permission X, X mainvars V'%action)foreid,expreid,expr,mainvarsinsession.execute(rql,build_descr=False):# we don't know yet if it's a rql expr for an entity or a relation,# so append a tuple to differentiate from groups and so we'll be# able to instantiate it laterres.setdefault(eid,{}).setdefault(action,[]).append((expr,mainvars,expreid))returnresdefset_perms(erschema,permsdict):"""set permissions on the given erschema according to the permission definition dictionary as built by deserialize_ertype_permissions for a given erschema's eid """try:thispermsdict=permsdict[erschema.eid]exceptKeyError:returnpermissions=erschema.permissionsforaction,somethingsinthispermsdict.iteritems():permissions[action]=tuple(isinstance(p,tuple)anderschema.rql_expression(*p)orpforpinsomethings)defdeserialize_rdef_constraints(session):"""return the list of relation definition's constraints as instances"""res={}forrdefeid,ceid,ct,valinsession.execute('Any E, X,TN,V WHERE E constrained_by X, X is CWConstraint, ''X cstrtype T, T name TN, X value V',build_descr=False):cstr=CONSTRAINTS[ct].deserialize(val)cstr.eid=ceidres.setdefault(rdefeid,[]).append(cstr)returnres# schema / perms serialization ################################################defserialize_schema(cursor,schema,verbose=False):"""synchronize schema and permissions in the database according to current schema """quiet=os.environ.get('APYCOT_ROOT')ifnotquiet:_title='-> storing the schema in the database 'print_title,execute=cursor.executeeschemas=schema.entities()aller=eschemas+schema.relations()ifnotverboseandnotquiet:pb_size=len(aller)+len(CONSTRAINTS)+len([xforxineschemasifx.specializes()])pb=ProgressBar(pb_size,title=_title)else:pb=Nonerql='INSERT CWConstraintType X: X name %(ct)s'forcstrtypeinCONSTRAINTS:ifverbose:printrqlexecute(rql,{'ct':unicode(cstrtype)},build_descr=False)ifpbisnotNone:pb.update()groupmap=group_mapping(cursor,interactive=False)forertypeinaller:# skip eid and has_text relationsifertypeinVIRTUAL_RTYPES:ifpbisnotNone:pb.update()continueforrql,kwargsinerschema2rql(schema[ertype],groupmap):ifverbose:printrql%kwargsexecute(rql,kwargs,build_descr=False)ifpbisnotNone:pb.update()forrql,kwargsinspecialize2rql(schema):ifverbose:printrql%kwargsexecute(rql,kwargs,build_descr=False)ifpbisnotNone:pb.update()ifnotquiet:printdef_ervalues(erschema):try:type_=unicode(erschema.type)exceptUnicodeDecodeError,e:raiseException("can't decode %s [was %s]"%(erschema.type,e))try:desc=unicode(erschema.description)oru''exceptUnicodeDecodeError,e:raiseException("can't decode %s [was %s]"%(erschema.description,e))return{'name':type_,'final':erschema.final,'description':desc,}defeschema_relations_values(eschema):values=_ervalues(eschema)relations=['X %s%%(%s)s'%(attr,attr)forattrinsorted(values)]returnrelations,values# XXX 2.47 migrationHAS_FULLTEXT_CONTAINER=Truedefrschema_relations_values(rschema):values=_ervalues(rschema)values['final']=rschema.finalvalues['symetric']=rschema.symetricvalues['inlined']=rschema.inlinedifHAS_FULLTEXT_CONTAINER:ifisinstance(rschema.fulltext_container,str):values['fulltext_container']=unicode(rschema.fulltext_container)else:values['fulltext_container']=rschema.fulltext_containerrelations=['X %s%%(%s)s'%(attr,attr)forattrinsorted(values)]returnrelations,valuesdef_rdef_values(rschema,objtype,props):amap={'order':'ordernum'}values={}forprop,defaultinschemamod.RelationDefinitionSchema.rproperty_defs(objtype).iteritems():ifpropin('eid','constraints','uid','infered','permissions'):continuevalue=props.get(prop,default)ifpropin('indexed','fulltextindexed','internationalizable'):value=bool(value)elifprop=='ordernum':value=int(value)elifisinstance(value,str):value=unicode(value)values[amap.get(prop,prop)]=valuereturnvaluesdefnfrdef_relations_values(rschema,objtype,props):values=_rdef_values(rschema,objtype,props)relations=['X %s%%(%s)s'%(attr,attr)forattrinsorted(values)]returnrelations,valuesdeffrdef_relations_values(rschema,objtype,props):values=_rdef_values(rschema,objtype,props)default=values['default']delvalues['default']ifdefaultisnotNone:ifdefaultisFalse:default=u''elifnotisinstance(default,unicode):default=unicode(default)values['defaultval']=defaultrelations=['X %s%%(%s)s'%(attr,attr)forattrinsorted(values)]returnrelations,valuesdef__rdef2rql(genmap,rschema,subjtype=None,objtype=None,props=None,groupmap=None):ifsubjtypeisNone:assertobjtypeisNoneassertpropsisNonetargets=sorted(rschema.rdefs)else:assertnotobjtypeisNonetargets=[(subjtype,objtype)]# relation schemaifrschema.final:etype='CWAttribute'else:etype='CWRelation'forsubjtype,objtypeintargets:ifpropsisNone:_props=rschema.rdef(subjtype,objtype)else:_props=props# don't serialize infered relationsif_props.get('infered'):continuegen=genmap[rschema.final]forrql,valuesingen(rschema,subjtype,objtype,_props):yieldrql,values# no groupmap means "no security insertion"ifgroupmap:forrql,argsin_erperms2rql(_props,groupmap):args['st']=str(subjtype)args['rt']=str(rschema)args['ot']=str(objtype)yieldrql+'X is %s, X from_entity ST, X to_entity OT, '\'X relation_type RT, RT name %%(rt)s, ST name %%(st)s, '\'OT name %%(ot)s'%etype,argsdefschema2rql(schema,skip=None,allow=None):"""return a list of rql insert statements to enter the schema in the database as CWRType and CWEType entities """assertnot(skipisnotNoneandallowisnotNone), \'can\'t use both skip and allow'all=schema.entities()+schema.relations()ifskipisnotNone:returnchain(*[erschema2rql(schema[t])fortinallifnottinskip])elifallowisnotNone:returnchain(*[erschema2rql(schema[t])fortinalliftinallow])returnchain(*[erschema2rql(schema[t])fortinall])deferschema2rql(erschema,groupmap):ifisinstance(erschema,schemamod.EntitySchema):returneschema2rql(erschema,groupmap=groupmap)returnrschema2rql(erschema,groupmap=groupmap)defeschema2rql(eschema,groupmap=None):"""return a list of rql insert statements to enter an entity schema in the database as an CWEType entity """relations,values=eschema_relations_values(eschema)# NOTE: 'specializes' relation can't be inserted here since there's no# way to make sure the parent type is inserted before the child typeyield'INSERT CWEType X: %s'%','.join(relations),values# entity permissionsifgroupmapisnotNone:forrql,argsin_erperms2rql(eschema,groupmap):args['name']=str(eschema)yieldrql+'X is CWEType, X name %(name)s',argsdefspecialize2rql(schema):foreschemainschema.entities():forrql,kwargsineschemaspecialize2rql(eschema):yieldrql,kwargsdefeschemaspecialize2rql(eschema):specialized_type=eschema.specializes()ifspecialized_type:values={'x':eschema.type,'et':specialized_type.type}yield'SET X specializes ET WHERE X name %(x)s, ET name %(et)s',valuesdefrschema2rql(rschema,addrdef=True,groupmap=None):"""return a list of rql insert statements to enter a relation schema in the database as an CWRType entity """ifrschema.type=='has_text':returnrelations,values=rschema_relations_values(rschema)yield'INSERT CWRType X: %s'%','.join(relations),valuesifaddrdef:forrql,valuesinrdef2rql(rschema,groupmap=groupmap):yieldrql,valuesdefrdef2rql(rschema,subjtype=None,objtype=None,props=None,groupmap=None):genmap={True:frdef2rql,False:nfrdef2rql}return__rdef2rql(genmap,rschema,subjtype,objtype,props,groupmap)_LOCATE_RDEF_RQL0='X relation_type ER,X from_entity SE,X to_entity OE'_LOCATE_RDEF_RQL1='SE name %(se)s,ER name %(rt)s,OE name %(oe)s'deffrdef2rql(rschema,subjtype,objtype,props):relations,values=frdef_relations_values(rschema,objtype,props)relations.append(_LOCATE_RDEF_RQL0)values.update({'se':str(subjtype),'rt':str(rschema),'oe':str(objtype)})yield'INSERT CWAttribute X: %s WHERE %s'%(','.join(relations),_LOCATE_RDEF_RQL1),valuesforrql,valuesinrdefrelations2rql(rschema,subjtype,objtype,props):yieldrql+', EDEF is CWAttribute',valuesdefnfrdef2rql(rschema,subjtype,objtype,props):relations,values=nfrdef_relations_values(rschema,objtype,props)relations.append(_LOCATE_RDEF_RQL0)values.update({'se':str(subjtype),'rt':str(rschema),'oe':str(objtype)})yield'INSERT CWRelation X: %s WHERE %s'%(','.join(relations),_LOCATE_RDEF_RQL1),valuesforrql,valuesinrdefrelations2rql(rschema,subjtype,objtype,props):yieldrql+', EDEF is CWRelation',valuesdefrdefrelations2rql(rschema,subjtype,objtype,props):iterators=[]forconstraintinprops.constraints:iterators.append(constraint2rql(rschema,subjtype,objtype,constraint))returnchain(*iterators)defconstraint2rql(rschema,subjtype,objtype,constraint):values={'ctname':unicode(constraint.type()),'value':unicode(constraint.serialize()),'rt':str(rschema),'se':str(subjtype),'oe':str(objtype)}yield'INSERT CWConstraint X: X value %(value)s, X cstrtype CT, EDEF constrained_by X WHERE \CT name %(ctname)s, EDEF relation_type ER, EDEF from_entity SE, EDEF to_entity OE, \ER name %(rt)s, SE name %(se)s, OE name %(oe)s',valuesdef_erperms2rql(erschema,groupmap):"""return rql insert statements to enter the entity or relation schema's permissions in the database as [read|add|delete|update]_permission relations between CWEType/CWRType and CWGroup entities """foractioninerschema.ACTIONS:forgroup_or_rqlexprinerschema.action_permissions(action):ifisinstance(group_or_rqlexpr,basestring):# grouptry:yield('SET X %s_permission Y WHERE Y eid %%(g)s, '%action,{'g':groupmap[group_or_rqlexpr]})exceptKeyError:continueelse:# rqlexprrqlexpr=group_or_rqlexpryield('INSERT RQLExpression E: E expression %%(e)s, E exprtype %%(t)s, ''E mainvars %%(v)s, X %s_permission E WHERE '%action,{'e':unicode(rqlexpr.expression),'v':unicode(rqlexpr.mainvars),'t':unicode(rqlexpr.__class__.__name__)})defupdateeschema2rql(eschema):relations,values=eschema_relations_values(eschema)values['et']=eschema.typeyield'SET %s WHERE X is CWEType, X name %%(et)s'%','.join(relations),valuesdefupdaterschema2rql(rschema):relations,values=rschema_relations_values(rschema)values['rt']=rschema.typeyield'SET %s WHERE X is CWRType, X name %%(rt)s'%','.join(relations),valuesdefupdaterdef2rql(rschema,subjtype=None,objtype=None,props=None):genmap={True:updatefrdef2rql,False:updatenfrdef2rql}return__rdef2rql(genmap,rschema,subjtype,objtype,props)defupdatefrdef2rql(rschema,subjtype,objtype,props):relations,values=frdef_relations_values(rschema,objtype,props)values.update({'se':subjtype,'rt':str(rschema),'oe':objtype})yield'SET %s WHERE %s, %s, X is CWAttribute'%(','.join(relations),_LOCATE_RDEF_RQL0,_LOCATE_RDEF_RQL1),valuesdefupdatenfrdef2rql(rschema,subjtype,objtype,props):relations,values=nfrdef_relations_values(rschema,objtype,props)values.update({'se':subjtype,'rt':str(rschema),'oe':objtype})yield'SET %s WHERE %s, %s, X is CWRelation'%(','.join(relations),_LOCATE_RDEF_RQL0,_LOCATE_RDEF_RQL1),values