schema serialization optimization by using eids instead of type names. Heavy refactoring/cleanup on the way
authorSylvain Thénault <sylvain.thenault@logilab.fr>
Wed, 03 Mar 2010 17:56:04 +0100
changeset 4763 81b0df087375
parent 4762 8dce25da9d95
child 4764 ec9c20c6b9f7
schema serialization optimization by using eids instead of type names. Heavy refactoring/cleanup on the way
hooks/syncschema.py
server/migractions.py
server/schemaserial.py
server/test/unittest_migractions.py
server/test/unittest_schemaserial.py
--- a/hooks/syncschema.py	Wed Mar 03 17:51:49 2010 +0100
+++ b/hooks/syncschema.py	Wed Mar 03 17:56:04 2010 +0100
@@ -12,11 +12,12 @@
 """
 __docformat__ = "restructuredtext en"
 
+from copy import copy
 from yams.schema import BASE_TYPES, RelationSchema, RelationDefinitionSchema
-from yams.buildobjs import EntityType, RelationType, RelationDefinition
-from yams.schema2sql import eschema2sql, rschema2sql, type_from_constraints
+from yams import buildobjs as ybo, schema2sql as y2sql
 
 from logilab.common.decorators import clear_cache
+from logilab.common.testlib import mock_object
 
 from cubicweb import ValidationError
 from cubicweb.selectors import implements
@@ -246,7 +247,7 @@
             # need to create the relation if it has not been already done by
             # another event of the same transaction
             if not rschema.type in session.transaction_data.get('createdtables', ()):
-                tablesql = rschema2sql(rschema)
+                tablesql = y2sql.rschema2sql(rschema)
                 # create the necessary table
                 for sql in tablesql.split(';'):
                     if sql.strip():
@@ -314,13 +315,13 @@
         rtype = entity.rtype.name
         obj = str(entity.otype.name)
         constraints = get_constraints(self.session, entity)
-        rdef = RelationDefinition(subj, rtype, obj,
-                                  description=entity.description,
-                                  cardinality=entity.cardinality,
-                                  constraints=constraints,
-                                  order=entity.ordernum,
-                                  eid=entity.eid,
-                                  **kwargs)
+        rdef = ybo.RelationDefinition(subj, rtype, obj,
+                                      description=entity.description,
+                                      cardinality=entity.cardinality,
+                                      constraints=constraints,
+                                      order=entity.ordernum,
+                                      eid=entity.eid,
+                                      **kwargs)
         MemSchemaRDefAdd(self.session, rdef)
         return rdef
 
@@ -338,8 +339,8 @@
                  'internationalizable': entity.internationalizable}
         rdef = self.init_rdef(**props)
         sysource = session.pool.source('system')
-        attrtype = type_from_constraints(sysource.dbhelper, rdef.object,
-                                         rdef.constraints)
+        attrtype = y2sql.type_from_constraints(
+            sysource.dbhelper, rdef.object, rdef.constraints)
         # XXX should be moved somehow into lgc.adbh: sqlite doesn't support to
         # add a new column with UNIQUE, it should be added after the ALTER TABLE
         # using ADD INDEX
@@ -370,12 +371,13 @@
                 self.error('error while creating index for %s.%s: %s',
                            table, column, ex)
         # final relations are not infered, propagate
+        schema = session.vreg.schema
         try:
-            eschema = session.vreg.schema.eschema(rdef.subject)
+            eschema = schema.eschema(rdef.subject)
         except KeyError:
             return # entity type currently being added
         # propagate attribute to children classes
-        rschema = session.vreg.schema.rschema(rdef.name)
+        rschema = schema.rschema(rdef.name)
         # if relation type has been inserted in the same transaction, its final
         # attribute is still set to False, so we've to ensure it's False
         rschema.final = True
@@ -385,15 +387,19 @@
                       'cardinality': rdef.cardinality,
                       'constraints': rdef.constraints,
                       'permissions': rdef.get_permissions(),
-                      'order': rdef.order})
+                      'order': rdef.order,
+                      'infered': False, 'eid': None
+                      })
+        cstrtypemap = ss.cstrtype_mapping(session)
         groupmap = group_mapping(session)
+        object = schema.eschema(rdef.object)
         for specialization in eschema.specialized_by(False):
             if (specialization, rdef.object) in rschema.rdefs:
                 continue
-            sperdef = RelationDefinitionSchema(specialization, rschema, rdef.object, props)
-            for rql, args in ss.rdef2rql(rschema, str(specialization),
-                                         rdef.object, sperdef, groupmap=groupmap):
-                session.execute(rql, args)
+            sperdef = RelationDefinitionSchema(specialization, rschema,
+                                               object, props)
+            ss.execschemarql(session.execute, sperdef,
+                             ss.rdef2rql(sperdef, cstrtypemap, groupmap))
         # set default value, using sql for performance and to avoid
         # modification_date update
         if default:
@@ -442,13 +448,13 @@
                     rtype in session.transaction_data.get('createdtables', ())):
                 try:
                     rschema = schema.rschema(rtype)
-                    tablesql = rschema2sql(rschema)
+                    tablesql = y2sql.rschema2sql(rschema)
                 except KeyError:
                     # fake we add it to the schema now to get a correctly
                     # initialized schema but remove it before doing anything
                     # more dangerous...
                     rschema = schema.add_relation_type(rdef)
-                    tablesql = rschema2sql(rschema)
+                    tablesql = y2sql.rschema2sql(rschema)
                     schema.del_relation_type(rtype)
                 # create the necessary table
                 for sql in tablesql.split(';'):
@@ -480,11 +486,11 @@
                 return
             atype = self.rschema.objects(etype)[0]
             constraints = self.rschema.rdef(etype, atype).constraints
-            coltype = type_from_constraints(adbh, atype, constraints,
-                                            creating=False)
+            coltype = y2sql.type_from_constraints(adbh, atype, constraints,
+                                                  creating=False)
             # XXX check self.values['cardinality'][0] actually changed?
-            sql = adbh.sql_set_null_allowed(table, column, coltype,
-                                            self.values['cardinality'][0] != '1')
+            notnull = self.values['cardinality'][0] != '1'
+            sql = adbh.sql_set_null_allowed(table, column, coltype, notnull)
             self.session.system_sql(sql)
         if 'fulltextindexed' in self.values:
             UpdateFTIndexOp(self.session)
@@ -517,8 +523,8 @@
             oldcstr is None or oldcstr.max != newcstr.max):
             adbh = self.session.pool.source('system').dbhelper
             card = rtype.rdef(subjtype, objtype).cardinality
-            coltype = type_from_constraints(adbh, objtype, [newcstr],
-                                            creating=False)
+            coltype = y2sql.type_from_constraints(adbh, objtype, [newcstr],
+                                                  creating=False)
             sql = adbh.sql_change_col_type(table, column, coltype, card != '1')
             try:
                 session.system_sql(sql, rollback_on_failure=False)
@@ -819,23 +825,26 @@
             return
         schema = self._cw.vreg.schema
         name = entity['name']
-        etype = EntityType(name=name, description=entity.get('description'),
-                           meta=entity.get('meta')) # don't care about final
+        etype = ybo.EntityType(name=name, description=entity.get('description'),
+                               meta=entity.get('meta')) # don't care about final
         # fake we add it to the schema now to get a correctly initialized schema
         # but remove it before doing anything more dangerous...
         schema = self._cw.vreg.schema
         eschema = schema.add_entity_type(etype)
         # generate table sql and rql to add metadata
-        tablesql = eschema2sql(self._cw.pool.source('system').dbhelper, eschema,
-                               prefix=SQL_PREFIX)
-        relrqls = []
+        tablesql = y2sql.eschema2sql(self._cw.pool.source('system').dbhelper,
+                                     eschema, prefix=SQL_PREFIX)
+        rdefrqls = []
+        gmap = group_mapping(self._cw)
+        cmap = ss.cstrtype_mapping(self._cw)
         for rtype in (META_RTYPES - VIRTUAL_RTYPES):
             rschema = schema[rtype]
             sampletype = rschema.subjects()[0]
             desttype = rschema.objects()[0]
-            props = rschema.rdef(sampletype, desttype)
-            relrqls += list(ss.rdef2rql(rschema, name, desttype, props,
-                                        groupmap=group_mapping(self._cw)))
+            rdef = copy(rschema.rdef(sampletype, desttype))
+            rdef.subject = mock_object(eid=entity.eid)
+            mock = mock_object(eid=None)
+            rdefrqls.append( (mock, tuple(ss.rdef2rql(rdef, cmap, gmap))) )
         # now remove it !
         schema.del_entity_type(name)
         # create the necessary table
@@ -848,8 +857,8 @@
         etype.eid = entity.eid
         MemSchemaCWETypeAdd(self._cw, etype)
         # add meta relations
-        for rql, kwargs in relrqls:
-            self._cw.execute(rql, kwargs)
+        for rdef, relrqls in rdefrqls:
+            ss.execschemarql(self._cw.execute, rdef, relrqls)
 
 
 class BeforeUpdateCWETypeHook(DelCWETypeHook):
@@ -906,12 +915,12 @@
 
     def __call__(self):
         entity = self.entity
-        rtype = RelationType(name=entity.name,
-                             description=entity.get('description'),
-                             meta=entity.get('meta', False),
-                             inlined=entity.get('inlined', False),
-                             symmetric=entity.get('symmetric', False),
-                             eid=entity.eid)
+        rtype = ybo.RelationType(name=entity.name,
+                                 description=entity.get('description'),
+                                 meta=entity.get('meta', False),
+                                 inlined=entity.get('inlined', False),
+                                 symmetric=entity.get('symmetric', False),
+                                 eid=entity.eid)
         MemSchemaCWRTypeAdd(self._cw, rtype)
 
 
--- a/server/migractions.py	Wed Mar 03 17:51:49 2010 +0100
+++ b/server/migractions.py	Wed Mar 03 17:56:04 2010 +0100
@@ -25,10 +25,12 @@
 import os.path as osp
 from datetime import datetime
 from glob import glob
+from copy import copy
 from warnings import warn
 
 from logilab.common.deprecation import deprecated
 from logilab.common.decorators import cached, clear_cache
+from logilab.common.testlib import mock_object
 
 from yams.constraints import SizeConstraint
 from yams.schema2sql import eschema2sql, rschema2sql
@@ -283,6 +285,11 @@
         """cached group mapping"""
         return ss.group_mapping(self._cw)
 
+    @cached
+    def cstrtype_mapping(self):
+        """cached constraint types mapping"""
+        return ss.cstrtype_mapping(self._cw)
+
     def exec_event_script(self, event, cubepath=None, funcname=None,
                           *args, **kwargs):
         if cubepath:
@@ -400,14 +407,17 @@
             return
         self._synchronized.add(rtype)
         rschema = self.fs_schema.rschema(rtype)
+        reporschema = self.repo.schema.rschema(rtype)
         if syncprops:
-            self.rqlexecall(ss.updaterschema2rql(rschema),
+            assert reporschema.eid, reporschema
+            self.rqlexecall(ss.updaterschema2rql(rschema, reporschema.eid),
                             ask_confirm=self.verbosity>=2)
         if syncrdefs:
-            reporschema = self.repo.schema.rschema(rtype)
             for subj, obj in rschema.rdefs:
                 if (subj, obj) not in reporschema.rdefs:
                     continue
+                if rschema in VIRTUAL_RTYPES:
+                    continue
                 self._synchronize_rdef_schema(subj, rschema, obj,
                                               syncprops=syncprops,
                                               syncperms=syncperms)
@@ -440,9 +450,11 @@
                          'Y is CWEType, Y name %(y)s',
                          {'x': str(repoeschema), 'y': str(espschema)},
                          ask_confirm=False)
-        self.rqlexecall(ss.updateeschema2rql(eschema),
+        self.rqlexecall(ss.updateeschema2rql(eschema, repoeschema.eid),
                         ask_confirm=self.verbosity >= 2)
         for rschema, targettypes, role in eschema.relation_definitions(True):
+            if rschema in VIRTUAL_RTYPES:
+                continue
             if role == 'subject':
                 if not rschema in repoeschema.subject_relations():
                     continue
@@ -480,11 +492,11 @@
         confirm = self.verbosity >= 2
         if syncprops:
             # properties
-            self.rqlexecall(ss.updaterdef2rql(rschema, subjtype, objtype),
+            rdef = rschema.rdef(subjtype, objtype)
+            repordef = reporschema.rdef(subjtype, objtype)
+            self.rqlexecall(ss.updaterdef2rql(rdef, repordef.eid),
                             ask_confirm=confirm)
             # constraints
-            rdef = rschema.rdef(subjtype, objtype)
-            repordef = reporschema.rdef(subjtype, objtype)
             newconstraints = list(rdef.constraints)
             # 1. remove old constraints and update constraints of the same type
             # NOTE: don't use rschema.constraint_by_type because it may be
@@ -510,10 +522,10 @@
                     self.rqlexec('SET X value %(v)s WHERE X eid %(x)s',
                                  values, 'x', ask_confirm=confirm)
             # 2. add new constraints
-            for newcstr in newconstraints:
-                self.rqlexecall(ss.constraint2rql(rschema, subjtype, objtype,
-                                                  newcstr),
-                                ask_confirm=confirm)
+            cstrtype_map = self.cstrtype_mapping()
+            self.rqlexecall(ss.constraints2rql(cstrtype_map, newconstraints,
+                                               repordef.eid),
+                            ask_confirm=confirm)
         if syncperms and not rschema in VIRTUAL_RTYPES:
             self._synchronize_permissions(rdef, repordef.eid)
 
@@ -674,18 +686,23 @@
         targeted type is known
         """
         instschema = self.repo.schema
-        if etype in instschema:
-            # XXX (syt) plz explain: if we're adding an entity type, it should
-            # not be there...
-            eschema = instschema[etype]
-            if eschema.final:
-                instschema.del_entity_type(etype)
-        else:
-            eschema = self.fs_schema.eschema(etype)
+        assert not etype in instschema
+        #     # XXX (syt) plz explain: if we're adding an entity type, it should
+        #     # not be there...
+        #     eschema = instschema[etype]
+        #     if eschema.final:
+        #         instschema.del_entity_type(etype)
+        # else:
+        eschema = self.fs_schema.eschema(etype)
         confirm = self.verbosity >= 2
         groupmap = self.group_mapping()
+        cstrtypemap = self.cstrtype_mapping()
         # register the entity into CWEType
-        self.rqlexecall(ss.eschema2rql(eschema, groupmap), ask_confirm=confirm)
+        try:
+            execute = self._cw.unsafe_execute
+        except AttributeError:
+            execute = self._cw.execute
+        ss.execschemarql(execute, eschema, ss.eschema2rql(eschema, groupmap))
         # add specializes relation if needed
         self.rqlexecall(ss.eschemaspecialize2rql(eschema), ask_confirm=confirm)
         # register entity's attributes
@@ -698,9 +715,8 @@
                 # actually in the schema
                 self.cmd_add_relation_type(rschema.type, False, commit=True)
             # register relation definition
-            self.rqlexecall(ss.rdef2rql(rschema, etype, attrschema.type,
-                                        groupmap=groupmap),
-                            ask_confirm=confirm)
+            rdef = self._get_rdef(rschema, eschema, eschema.destination(rschema))
+            ss.execschemarql(execute, rdef, ss.rdef2rql(rdef, cstrtypemap, groupmap),)
         # take care to newly introduced base class
         # XXX some part of this should probably be under the "if auto" block
         for spschema in eschema.specialized_by(recursive=False):
@@ -760,10 +776,12 @@
                     # remember this two avoid adding twice non symmetric relation
                     # such as "Emailthread forked_from Emailthread"
                     added.append((etype, rschema.type, targettype))
-                    self.rqlexecall(ss.rdef2rql(rschema, etype, targettype,
-                                                groupmap=groupmap),
-                                    ask_confirm=confirm)
+                    rdef = self._get_rdef(rschema, eschema, targetschema)
+                    ss.execschemarql(execute, rdef,
+                                     ss.rdef2rql(rdef, cstrtypemap, groupmap))
             for rschema in eschema.object_relations():
+                if rschema.type in META_RTYPES:
+                    continue
                 rtypeadded = rschema.type in instschema or rschema.type in added
                 for targetschema in rschema.subjects(etype):
                     # ignore relations where the targeted type is not in the
@@ -781,9 +799,9 @@
                     elif (targettype, rschema.type, etype) in added:
                         continue
                     # register relation definition
-                    self.rqlexecall(ss.rdef2rql(rschema, targettype, etype,
-                                                groupmap=groupmap),
-                                    ask_confirm=confirm)
+                    rdef = self._get_rdef(rschema, targetschema, eschema)
+                    ss.execschemarql(execute, rdef,
+                                     ss.rdef2rql(rdef, cstrtypemap, groupmap))
         if commit:
             self.commit()
 
@@ -822,15 +840,26 @@
         committing depends on the `commit` argument value).
 
         """
+        reposchema = self.repo.schema
         rschema = self.fs_schema.rschema(rtype)
+        try:
+            execute = self._cw.unsafe_execute
+        except AttributeError:
+            execute = self._cw.execute
         # register the relation into CWRType and insert necessary relation
         # definitions
-        self.rqlexecall(ss.rschema2rql(rschema, addrdef=False),
-                        ask_confirm=self.verbosity>=2)
+        ss.execschemarql(execute, rschema, ss.rschema2rql(rschema, addrdef=False))
         if addrdef:
             self.commit()
-            self.rqlexecall(ss.rdef2rql(rschema, groupmap=self.group_mapping()),
-                            ask_confirm=self.verbosity>=2)
+            gmap = self.group_mapping()
+            cmap = self.cstrtype_mapping()
+            for rdef in rschema.rdefs.itervalues():
+                if not (reposchema.has_entity(rdef.subject)
+                        and reposchema.has_entity(rdef.object)):
+                    continue
+                self._set_rdef_eid(rdef)
+                ss.execschemarql(execute, rdef,
+                                 ss.rdef2rql(rdef, cmap, gmap))
             if rtype in META_RTYPES:
                 # if the relation is in META_RTYPES, ensure we're adding it for
                 # all entity types *in the persistent schema*, not only those in
@@ -839,15 +868,14 @@
                     if not etype in self.fs_schema:
                         # get sample object type and rproperties
                         objtypes = rschema.objects()
-                        assert len(objtypes) == 1
+                        assert len(objtypes) == 1, objtypes
                         objtype = objtypes[0]
-                        props = rschema.rproperties(
-                            rschema.subjects(objtype)[0], objtype)
-                        assert props
-                        self.rqlexecall(ss.rdef2rql(rschema, etype, objtype, props,
-                                                    groupmap=self.group_mapping()),
-                                        ask_confirm=self.verbosity>=2)
-
+                        rdef = copy(rschema.rdef(rschema.subjects(objtype)[0], objtype))
+                        rdef.subject = etype
+                        rdef.rtype = self.repo.schema.rschema(rschema)
+                        rdef.object = self.repo.schema.rschema(objtype)
+                        ss.execschemarql(execute, rdef,
+                                         ss.rdef2rql(rdef, cmap, gmap))
         if commit:
             self.commit()
 
@@ -877,12 +905,28 @@
         rschema = self.fs_schema.rschema(rtype)
         if not rtype in self.repo.schema:
             self.cmd_add_relation_type(rtype, addrdef=False, commit=True)
-        self.rqlexecall(ss.rdef2rql(rschema, subjtype, objtype,
-                                    groupmap=self.group_mapping()),
-                        ask_confirm=self.verbosity>=2)
+        try:
+            execute = self._cw.unsafe_execute
+        except AttributeError:
+            execute = self._cw.execute
+        rdef = self._get_rdef(rschema, subjtype, objtype)
+        ss.execschemarql(execute, rdef,
+                         ss.rdef2rql(rdef, self.cstrtype_mapping(),
+                                     self.group_mapping()))
         if commit:
             self.commit()
 
+    def _get_rdef(self, rschema, subjtype, objtype):
+        return self._set_rdef_eid(rschema.rdefs[(subjtype, objtype)])
+
+    def _set_rdef_eid(self, rdef):
+        for attr in ('rtype', 'subject', 'object'):
+            schemaobj = getattr(rdef, attr)
+            if getattr(schemaobj, 'eid', None) is None:
+                schemaobj.eid =  self.repo.schema[schemaobj].eid
+                assert schemaobj.eid is not None
+        return rdef
+
     def cmd_drop_relation_definition(self, subjtype, rtype, objtype, commit=True):
         """unregister an existing relation definition"""
         rschema = self.repo.schema.rschema(rtype)
--- a/server/schemaserial.py	Wed Mar 03 17:51:49 2010 +0100
+++ b/server/schemaserial.py	Wed Mar 03 17:56:04 2010 +0100
@@ -50,6 +50,10 @@
                     continue
     return res
 
+def cstrtype_mapping(cursor):
+    """cached constraint types mapping"""
+    return dict(cursor.execute('Any T, X WHERE X is CWConstraintType, X name T'))
+
 # schema / perms deserialization ##############################################
 def deserialize_schema(schema, session):
     """return a schema according to information stored in an rql database
@@ -221,6 +225,7 @@
         pb = ProgressBar(pb_size, title=_title)
     else:
         pb = None
+    groupmap = group_mapping(cursor, interactive=False)
     # serialize all entity types, assuring CWEType is serialized first
     eschemas.remove(schema.eschema('CWEType'))
     eschemas.insert(0, schema.eschema('CWEType'))
@@ -229,9 +234,11 @@
         if pb is not None:
             pb.update()
     # serialize constraint types
+    cstrtypemap = {}
     rql = 'INSERT CWConstraintType X: X name %(ct)s'
     for cstrtype in CONSTRAINTS:
-        execute(rql, {'ct': unicode(cstrtype)}, build_descr=False)
+        cstrtypemap[cstrtype] = execute(rql, {'ct': unicode(cstrtype)},
+                                        build_descr=False)[0][0]
         if pb is not None:
             pb.update()
     # serialize relations
@@ -241,8 +248,15 @@
             if pb is not None:
                 pb.update()
             continue
-        for rql, kwargs in erschema2rql(schema[ertype], groupmap):
-            execute(rql, kwargs, build_descr=False)
+        execschemarql(execute, rschema, rschema2rql(rschema, addrdef=False))
+        if rschema.symmetric:
+            rdefs = [rdef for k, rdef in rschema.rdefs.iteritems()
+                     if (rdef.subject, rdef.object) == k]
+        else:
+            rdefs = rschema.rdefs.itervalues()
+        for rdef in rdefs:
+            execschemarql(execute, rdef,
+                          rdef2rql(rdef, cstrtypemap, groupmap))
         if pb is not None:
             pb.update()
     for rql, kwargs in specialize2rql(schema):
@@ -253,6 +267,55 @@
         print
 
 
+# high level serialization functions
+
+def execschemarql(execute, schema, rqls):
+    for rql, kwargs in rqls:
+        kwargs['x'] = schema.eid
+        rset = execute(rql, kwargs, build_descr=False)
+        if schema.eid is None:
+            schema.eid = rset[0][0]
+        else:
+            assert rset
+
+def erschema2rql(erschema, groupmap):
+    if isinstance(erschema, schemamod.EntitySchema):
+        return eschema2rql(erschema, groupmap=groupmap)
+    return rschema2rql(erschema, groupmap=groupmap)
+
+def specialize2rql(schema):
+    for eschema in schema.entities():
+        if eschema.final:
+            continue
+        for rql, kwargs in eschemaspecialize2rql(eschema):
+            yield rql, kwargs
+
+# etype serialization
+
+def eschema2rql(eschema, groupmap=None):
+    """return a list of rql insert statements to enter an entity schema
+    in the database as an CWEType entity
+    """
+    relations, values = eschema_relations_values(eschema)
+    # NOTE: 'specializes' relation can't be inserted here since there's no
+    # way to make sure the parent type is inserted before the child type
+    yield 'INSERT CWEType X: %s' % ','.join(relations) , values
+    # entity permissions
+    if groupmap is not None:
+        for rql, args in _erperms2rql(eschema, groupmap):
+            yield rql, args
+
+def eschema_relations_values(eschema):
+    values = _ervalues(eschema)
+    relations = ['X %s %%(%s)s' % (attr, attr) for attr in sorted(values)]
+    return relations, values
+
+def eschemaspecialize2rql(eschema):
+    specialized_type = eschema.specializes()
+    if specialized_type:
+        values = {'x': eschema.eid, 'et': specialized_type.eid}
+        yield 'SET X specializes ET WHERE X eid %(x)s, ET eid %(et)s', values
+
 def _ervalues(erschema):
     try:
         type_ = unicode(erschema.type)
@@ -268,10 +331,23 @@
         'description': desc,
         }
 
-def eschema_relations_values(eschema):
-    values = _ervalues(eschema)
-    relations = ['X %s %%(%s)s' % (attr, attr) for attr in sorted(values)]
-    return relations, values
+# rtype serialization
+
+def rschema2rql(rschema, cstrtypemap=None, addrdef=True, groupmap=None):
+    """return a list of rql insert statements to enter a relation schema
+    in the database as an CWRType entity
+    """
+    if rschema.type == 'has_text':
+        return
+    relations, values = rschema_relations_values(rschema)
+    yield 'INSERT CWRType X: %s' % ','.join(relations), values
+    if addrdef:
+        assert cstrtypemap
+        # sort for testing purpose
+        for rdef in sorted(rschema.rdefs.itervalues(),
+                           key=lambda x: (x.subject, x.object)):
+            for rql, values in rdef2rql(rdef, cstrtypemap, groupmap):
+                yield rql, values
 
 def rschema_relations_values(rschema):
     values = _ervalues(rschema)
@@ -285,169 +361,58 @@
     relations = ['X %s %%(%s)s' % (attr, attr) for attr in sorted(values)]
     return relations, values
 
-def _rdef_values(objtype, props):
-    amap = {'order': 'ordernum'}
+# rdef serialization
+
+def rdef2rql(rdef, cstrtypemap, groupmap=None):
+    # don't serialize infered relations
+    if rdef.infered:
+        return
+    relations, values = _rdef_values(rdef)
+    relations.append('X relation_type ER,X from_entity SE,X to_entity OE')
+    values.update({'se': rdef.subject.eid, 'rt': rdef.rtype.eid, 'oe': rdef.object.eid})
+    if rdef.final:
+        etype = 'CWAttribute'
+    else:
+        etype = 'CWRelation'
+    yield 'INSERT %s X: %s WHERE SE eid %%(se)s,ER eid %%(rt)s,OE eid %%(oe)s' % (
+        etype, ','.join(relations), ), values
+    for rql, values in constraints2rql(cstrtypemap, rdef.constraints):
+        yield rql, values
+    # no groupmap means "no security insertion"
+    if groupmap:
+        for rql, args in _erperms2rql(rdef, groupmap):
+            yield rql, args
+
+def _rdef_values(rdef):
+    amap = {'order': 'ordernum', 'default': 'defaultval'}
     values = {}
-    for prop, default in schemamod.RelationDefinitionSchema.rproperty_defs(objtype).iteritems():
+    for prop, default in rdef.rproperty_defs(rdef.object).iteritems():
         if prop in ('eid', 'constraints', 'uid', 'infered', 'permissions'):
             continue
-        value = props.get(prop, default)
+        value = getattr(rdef, prop)
+        # XXX type cast really necessary?
         if prop in ('indexed', 'fulltextindexed', 'internationalizable'):
             value = bool(value)
         elif prop == 'ordernum':
             value = int(value)
         elif isinstance(value, str):
             value = unicode(value)
+        if value is not None and prop == 'default':
+            if value is False:
+                value = u''
+            if not isinstance(value, unicode):
+                value = unicode(value)
         values[amap.get(prop, prop)] = value
-    return values
-
-def nfrdef_relations_values(objtype, props):
-    values = _rdef_values(objtype, props)
-    relations = ['X %s %%(%s)s' % (attr, attr) for attr in sorted(values)]
-    return relations, values
-
-def frdef_relations_values(objtype, props):
-    values = _rdef_values(objtype, props)
-    default = values['default']
-    del values['default']
-    if default is not None:
-        if default is False:
-            default = u''
-        elif not isinstance(default, unicode):
-            default = unicode(default)
-    values['defaultval'] = default
     relations = ['X %s %%(%s)s' % (attr, attr) for attr in sorted(values)]
     return relations, values
 
-
-def __rdef2rql(genmap, rschema, subjtype=None, objtype=None, props=None,
-               groupmap=None):
-    if subjtype is None:
-        assert objtype is None
-        assert props is None
-        targets = sorted(rschema.rdefs)
-    else:
-        assert not objtype is None
-        targets = [(subjtype, objtype)]
-    # relation schema
-    if rschema.final:
-        etype = 'CWAttribute'
-    else:
-        etype = 'CWRelation'
-    for subjtype, objtype in targets:
-        if props is None:
-            _props = rschema.rdef(subjtype, objtype)
-        else:
-            _props = props
-        # don't serialize infered relations
-        if _props.get('infered'):
-            continue
-        gen = genmap[rschema.final]
-        for rql, values in gen(rschema, subjtype, objtype, _props):
-            yield rql, values
-        # no groupmap means "no security insertion"
-        if groupmap:
-            for rql, args in _erperms2rql(_props, groupmap):
-                args['st'] = str(subjtype)
-                args['rt'] = str(rschema)
-                args['ot'] = str(objtype)
-                yield rql + 'X is %s, X from_entity ST, X to_entity OT, '\
-                      'X relation_type RT, RT name %%(rt)s, ST name %%(st)s, '\
-                      'OT name %%(ot)s' % etype, args
-
-
-def schema2rql(schema, skip=None, allow=None):
-    """return a list of rql insert statements to enter the schema in the
-    database as CWRType and CWEType entities
-    """
-    assert not (skip is not None and allow is not None), \
-           'can\'t use both skip and allow'
-    all = schema.entities() + schema.relations()
-    if skip is not None:
-        return chain(*[erschema2rql(schema[t]) for t in all if not t in skip])
-    elif allow is not None:
-        return chain(*[erschema2rql(schema[t]) for t in all if t in allow])
-    return chain(*[erschema2rql(schema[t]) for t in all])
-
-def erschema2rql(erschema, groupmap):
-    if isinstance(erschema, schemamod.EntitySchema):
-        return eschema2rql(erschema, groupmap=groupmap)
-    return rschema2rql(erschema, groupmap=groupmap)
-
-def eschema2rql(eschema, groupmap=None):
-    """return a list of rql insert statements to enter an entity schema
-    in the database as an CWEType entity
-    """
-    relations, values = eschema_relations_values(eschema)
-    # NOTE: 'specializes' relation can't be inserted here since there's no
-    # way to make sure the parent type is inserted before the child type
-    yield 'INSERT CWEType X: %s' % ','.join(relations) , values
-    # entity permissions
-    if groupmap is not None:
-        for rql, args in _erperms2rql(eschema, groupmap):
-            args['name'] = str(eschema)
-            yield rql + 'X is CWEType, X name %(name)s', args
-
-def specialize2rql(schema):
-    for eschema in schema.entities():
-        for rql, kwargs in eschemaspecialize2rql(eschema):
-            yield rql, kwargs
-
-def eschemaspecialize2rql(eschema):
-    specialized_type = eschema.specializes()
-    if specialized_type:
-        values = {'x': eschema.type, 'et': specialized_type.type}
-        yield 'SET X specializes ET WHERE X name %(x)s, ET name %(et)s', values
-
-def rschema2rql(rschema, addrdef=True, groupmap=None):
-    """return a list of rql insert statements to enter a relation schema
-    in the database as an CWRType entity
-    """
-    if rschema.type == 'has_text':
-        return
-    relations, values = rschema_relations_values(rschema)
-    yield 'INSERT CWRType X: %s' % ','.join(relations), values
-    if addrdef:
-        for rql, values in rdef2rql(rschema, groupmap=groupmap):
-            yield rql, values
-
-def rdef2rql(rschema, subjtype=None, objtype=None, props=None, groupmap=None):
-    genmap = {True: frdef2rql, False: nfrdef2rql}
-    return __rdef2rql(genmap, rschema, subjtype, objtype, props, groupmap)
-
-
-_LOCATE_RDEF_RQL0 = 'X relation_type ER,X from_entity SE,X to_entity OE'
-_LOCATE_RDEF_RQL1 = 'SE name %(se)s,ER name %(rt)s,OE name %(oe)s'
-
-def frdef2rql(rschema, subjtype, objtype, props):
-    relations, values = frdef_relations_values(objtype, props)
-    relations.append(_LOCATE_RDEF_RQL0)
-    values.update({'se': str(subjtype), 'rt': str(rschema), 'oe': str(objtype)})
-    yield 'INSERT CWAttribute X: %s WHERE %s' % (','.join(relations), _LOCATE_RDEF_RQL1), values
-    for rql, values in rdefrelations2rql(rschema, subjtype, objtype, props):
-        yield rql + ', EDEF is CWAttribute', values
-
-def nfrdef2rql(rschema, subjtype, objtype, props):
-    relations, values = nfrdef_relations_values(objtype, props)
-    relations.append(_LOCATE_RDEF_RQL0)
-    values.update({'se': str(subjtype), 'rt': str(rschema), 'oe': str(objtype)})
-    yield 'INSERT CWRelation X: %s WHERE %s' % (','.join(relations), _LOCATE_RDEF_RQL1), values
-    for rql, values in rdefrelations2rql(rschema, subjtype, objtype, props):
-        yield rql + ', EDEF is CWRelation', values
-
-def rdefrelations2rql(rschema, subjtype, objtype, props):
-    iterators = []
-    for constraint in props.constraints:
-        iterators.append(constraint2rql(rschema, subjtype, objtype, constraint))
-    return chain(*iterators)
-
-def constraint2rql(rschema, subjtype, objtype, constraint):
-    values = {'ctname': unicode(constraint.type()),
-              'value': unicode(constraint.serialize()),
-              'rt': str(rschema), 'se': str(subjtype), 'oe': str(objtype)}
-    yield 'INSERT CWConstraint X: X value %(value)s, X cstrtype CT, EDEF constrained_by X WHERE \
-CT name %(ctname)s, EDEF relation_type ER, EDEF from_entity SE, EDEF to_entity OE, \
-ER name %(rt)s, SE name %(se)s, OE name %(oe)s', values
+def constraints2rql(cstrtypemap, constraints, rdefeid=None):
+    for constraint in constraints:
+        values = {'ct': cstrtypemap[constraint.type()],
+                  'value': unicode(constraint.serialize()),
+                  'x': rdefeid} # when not specified, will have to be set by the caller
+        yield 'INSERT CWConstraint X: X value %(value)s, X cstrtype CT, EDEF constrained_by X WHERE \
+CT eid %(ct)s, EDEF eid %(x)s', values
 
 
 def _erperms2rql(erschema, groupmap):
@@ -466,7 +431,7 @@
             if isinstance(group_or_rqlexpr, basestring):
                 # group
                 try:
-                    yield ('SET X %s_permission Y WHERE Y eid %%(g)s, ' % action,
+                    yield ('SET X %s_permission Y WHERE Y eid %%(g)s, X eid %%(x)s' % action,
                            {'g': groupmap[group_or_rqlexpr]})
                 except KeyError:
                     continue
@@ -474,36 +439,24 @@
                 # rqlexpr
                 rqlexpr = group_or_rqlexpr
                 yield ('INSERT RQLExpression E: E expression %%(e)s, E exprtype %%(t)s, '
-                       'E mainvars %%(v)s, X %s_permission E WHERE ' % action,
+                       'E mainvars %%(v)s, X %s_permission E WHERE X eid %%(x)s' % action,
                        {'e': unicode(rqlexpr.expression),
                         'v': unicode(rqlexpr.mainvars),
                         't': unicode(rqlexpr.__class__.__name__)})
 
+# update functions
 
-def updateeschema2rql(eschema):
+def updateeschema2rql(eschema, eid):
     relations, values = eschema_relations_values(eschema)
-    values['et'] = eschema.type
-    yield 'SET %s WHERE X is CWEType, X name %%(et)s' % ','.join(relations), values
-
-def updaterschema2rql(rschema):
-    relations, values = rschema_relations_values(rschema)
-    values['rt'] = rschema.type
-    yield 'SET %s WHERE X is CWRType, X name %%(rt)s' % ','.join(relations), values
+    values['x'] = eid
+    yield 'SET %s WHERE X eid %%(x)s' % ','.join(relations), values
 
-def updaterdef2rql(rschema, subjtype=None, objtype=None, props=None):
-    genmap = {True: updatefrdef2rql, False: updatenfrdef2rql}
-    return __rdef2rql(genmap, rschema, subjtype, objtype, props)
+def updaterschema2rql(rschema, eid):
+    relations, values = rschema_relations_values(rschema)
+    values['x'] = eid
+    yield 'SET %s WHERE X eid %%(x)s' % ','.join(relations), values
 
-def updatefrdef2rql(rschema, subjtype, objtype, props):
-    relations, values = frdef_relations_values(objtype, props)
-    values.update({'se': subjtype, 'rt': str(rschema), 'oe': objtype})
-    yield 'SET %s WHERE %s, %s, X is CWAttribute' % (','.join(relations),
-                                                     _LOCATE_RDEF_RQL0,
-                                                     _LOCATE_RDEF_RQL1), values
-
-def updatenfrdef2rql(rschema, subjtype, objtype, props):
-    relations, values = nfrdef_relations_values(objtype, props)
-    values.update({'se': subjtype, 'rt': str(rschema), 'oe': objtype})
-    yield 'SET %s WHERE %s, %s, X is CWRelation' % (','.join(relations),
-                                                    _LOCATE_RDEF_RQL0,
-                                                    _LOCATE_RDEF_RQL1), values
+def updaterdef2rql(rdef, eid):
+    relations, values = _rdef_values(rdef)
+    values['x'] = eid
+    yield 'SET %s WHERE X eid %%(x)s' % ','.join(relations), values
--- a/server/test/unittest_migractions.py	Wed Mar 03 17:51:49 2010 +0100
+++ b/server/test/unittest_migractions.py	Wed Mar 03 17:56:04 2010 +0100
@@ -35,6 +35,13 @@
     def _refresh_repo(cls):
         super(MigrationCommandsTC, cls)._refresh_repo()
         cls.repo.set_schema(deepcopy(cls.origschema), resetvreg=False)
+        # reset migration schema eids
+        for eschema in migrschema.entities():
+            eschema.eid = None
+        for rschema in migrschema.relations():
+            rschema.eid = None
+            for rdef in rschema.rdefs.values():
+                rdef.eid = None
 
     def setUp(self):
         CubicWebTC.setUp(self)
@@ -44,7 +51,6 @@
         assert self.cnx is self.mh._cnx
         assert self.session is self.mh.session, (self.session.id, self.mh.session.id)
 
-
     def test_add_attribute_int(self):
         self.failIf('whatever' in self.schema)
         self.request().create_entity('Note')
--- a/server/test/unittest_schemaserial.py	Wed Mar 03 17:51:49 2010 +0100
+++ b/server/test/unittest_schemaserial.py	Wed Mar 03 17:56:04 2010 +0100
@@ -18,6 +18,12 @@
 from cubicweb.server.schemaserial import *
 from cubicweb.server.schemaserial import _erperms2rql as erperms2rql
 
+cstrtypemap = {'RQLConstraint': 'RQLConstraint_eid',
+               'SizeConstraint': 'SizeConstraint_eid',
+               'StaticVocabularyConstraint': 'StaticVocabularyConstraint_eid',
+               'FormatConstraint': 'FormatConstraint_eid',
+               }
+
 class Schema2RQLTC(TestCase):
 
     def test_eschema2rql1(self):
@@ -34,104 +40,124 @@
                  {'description': u'', 'final': True, 'name': u'String'})])
 
     def test_eschema2rql_specialization(self):
+        # x: None since eschema.eid are None
         self.assertListEquals(sorted(specialize2rql(schema)),
-                              [('SET X specializes ET WHERE X name %(x)s, ET name %(et)s',
-                                {'et': 'BaseTransition', 'x': 'Transition'}),
-                               ('SET X specializes ET WHERE X name %(x)s, ET name %(et)s',
-                                {'et': 'BaseTransition', 'x': 'WorkflowTransition'}),
-                               ('SET X specializes ET WHERE X name %(x)s, ET name %(et)s',
-                                {'et': 'Division', 'x': 'SubDivision'}),
-                               # ('SET X specializes ET WHERE X name %(x)s, ET name %(et)s',
+                              [('SET X specializes ET WHERE X eid %(x)s, ET eid %(et)s',
+                                {'et': None, 'x': None}),
+                               ('SET X specializes ET WHERE X eid %(x)s, ET eid %(et)s',
+                                {'et': None, 'x': None}),
+                               ('SET X specializes ET WHERE X eid %(x)s, ET eid %(et)s',
+                                {'et': None, 'x': None}),
+                               # ('SET X specializes ET WHERE X eid %(x)s, ET eid %(et)s',
                                #  {'et': 'File', 'x': 'Image'}),
-                               ('SET X specializes ET WHERE X name %(x)s, ET name %(et)s',
-                                {'et': 'Societe', 'x': 'Division'})])
+                               ('SET X specializes ET WHERE X eid %(x)s, ET eid %(et)s',
+                                {'et': None, 'x': None})])
 
     def test_rschema2rql1(self):
-        self.assertListEquals(list(rschema2rql(schema.rschema('relation_type'))),
+        self.assertListEquals(list(rschema2rql(schema.rschema('relation_type'), cstrtypemap)),
                              [
             ('INSERT CWRType X: X description %(description)s,X final %(final)s,X fulltext_container %(fulltext_container)s,X inlined %(inlined)s,X name %(name)s,X symmetric %(symmetric)s',
              {'description': u'link a relation definition to its relation type', 'symmetric': False, 'name': u'relation_type', 'final' : False, 'fulltext_container': None, 'inlined': True}),
 
-            ('INSERT CWRelation X: X cardinality %(cardinality)s,X composite %(composite)s,X description %(description)s,X ordernum %(ordernum)s,X relation_type ER,X from_entity SE,X to_entity OE WHERE SE name %(se)s,ER name %(rt)s,OE name %(oe)s',
-             {'rt': 'relation_type', 'description': u'', 'composite': u'object', 'oe': 'CWRType',
-              'ordernum': 1, 'cardinality': u'1*', 'se': 'CWAttribute'}),
-            ('INSERT CWConstraint X: X value %(value)s, X cstrtype CT, EDEF constrained_by X WHERE CT name %(ctname)s, EDEF relation_type ER, EDEF from_entity SE, EDEF to_entity OE, ER name %(rt)s, SE name %(se)s, OE name %(oe)s, EDEF is CWRelation',
-             {'rt': 'relation_type', 'oe': 'CWRType', 'ctname': u'RQLConstraint', 'se': 'CWAttribute', 'value': u';O;O final TRUE\n'}),
+            ('INSERT CWRelation X: X cardinality %(cardinality)s,X composite %(composite)s,X description %(description)s,X ordernum %(ordernum)s,X relation_type ER,X from_entity SE,X to_entity OE WHERE SE eid %(se)s,ER eid %(rt)s,OE eid %(oe)s',
+             {'se': None, 'rt': None, 'oe': None,
+              'description': u'', 'composite': u'object', 'cardinality': u'1*',
+              'ordernum': 1}),
+            ('INSERT CWConstraint X: X value %(value)s, X cstrtype CT, EDEF constrained_by X WHERE CT eid %(ct)s, EDEF eid %(x)s',
+             {'x': None, 'ct': u'RQLConstraint_eid', 'value': u';O;O final TRUE\n'}),
 
-            ('INSERT CWRelation X: X cardinality %(cardinality)s,X composite %(composite)s,X description %(description)s,X ordernum %(ordernum)s,X relation_type ER,X from_entity SE,X to_entity OE WHERE SE name %(se)s,ER name %(rt)s,OE name %(oe)s',
-             {'rt': 'relation_type', 'description': u'', 'composite': u'object', 'oe': 'CWRType',
-              'ordernum': 1, 'cardinality': u'1*', 'se': 'CWRelation'}),
-            ('INSERT CWConstraint X: X value %(value)s, X cstrtype CT, EDEF constrained_by X WHERE CT name %(ctname)s, EDEF relation_type ER, EDEF from_entity SE, EDEF to_entity OE, ER name %(rt)s, SE name %(se)s, OE name %(oe)s, EDEF is CWRelation',
-             {'rt': 'relation_type', 'oe': 'CWRType', 'ctname': u'RQLConstraint', 'se': 'CWRelation', 'value': u';O;O final FALSE\n'}),
+            ('INSERT CWRelation X: X cardinality %(cardinality)s,X composite %(composite)s,X description %(description)s,X ordernum %(ordernum)s,X relation_type ER,X from_entity SE,X to_entity OE WHERE SE eid %(se)s,ER eid %(rt)s,OE eid %(oe)s',
+             {'se': None, 'rt': None, 'oe': None,
+              'description': u'', 'composite': u'object', 
+              'ordernum': 1, 'cardinality': u'1*'}),
+            ('INSERT CWConstraint X: X value %(value)s, X cstrtype CT, EDEF constrained_by X WHERE CT eid %(ct)s, EDEF eid %(x)s',
+             {'x': None, 'ct': u'RQLConstraint_eid', 'value': u';O;O final FALSE\n'}),
             ])
 
     def test_rschema2rql2(self):
-        self.assertListEquals(list(rschema2rql(schema.rschema('add_permission'))),
+        self.assertListEquals(list(rschema2rql(schema.rschema('add_permission'), cstrtypemap)),
                               [
             ('INSERT CWRType X: X description %(description)s,X final %(final)s,X fulltext_container %(fulltext_container)s,X inlined %(inlined)s,X name %(name)s,X symmetric %(symmetric)s', {'description': u'', 'symmetric': False, 'name': u'add_permission', 'final': False, 'fulltext_container': None, 'inlined': False}),
 
-            ('INSERT CWRelation X: X cardinality %(cardinality)s,X composite %(composite)s,X description %(description)s,X ordernum %(ordernum)s,X relation_type ER,X from_entity SE,X to_entity OE WHERE SE name %(se)s,ER name %(rt)s,OE name %(oe)s',
-             {'rt': 'add_permission', 'description': u'groups allowed to add entities/relations of this type', 'composite': None, 'oe': 'CWGroup', 'ordernum': 9999, 'cardinality': u'**', 'se': 'CWEType'}),
-            ('INSERT CWRelation X: X cardinality %(cardinality)s,X composite %(composite)s,X description %(description)s,X ordernum %(ordernum)s,X relation_type ER,X from_entity SE,X to_entity OE WHERE SE name %(se)s,ER name %(rt)s,OE name %(oe)s',
-             {'rt': 'add_permission', 'description': u'rql expression allowing to add entities/relations of this type', 'composite': 'subject', 'oe': 'RQLExpression', 'ordernum': 9999, 'cardinality': u'*?', 'se': 'CWEType'}),
+            ('INSERT CWRelation X: X cardinality %(cardinality)s,X composite %(composite)s,X description %(description)s,X ordernum %(ordernum)s,X relation_type ER,X from_entity SE,X to_entity OE WHERE SE eid %(se)s,ER eid %(rt)s,OE eid %(oe)s',
+             {'se': None, 'rt': None, 'oe': None,
+              'description': u'groups allowed to add entities/relations of this type', 'composite': None, 'ordernum': 9999, 'cardinality': u'**'}),
+            ('INSERT CWRelation X: X cardinality %(cardinality)s,X composite %(composite)s,X description %(description)s,X ordernum %(ordernum)s,X relation_type ER,X from_entity SE,X to_entity OE WHERE SE eid %(se)s,ER eid %(rt)s,OE eid %(oe)s',
+             {'se': None, 'rt': None, 'oe': None,
+              'description': u'rql expression allowing to add entities/relations of this type', 'composite': 'subject', 'ordernum': 9999, 'cardinality': u'*?'}),
 
-            ('INSERT CWRelation X: X cardinality %(cardinality)s,X composite %(composite)s,X description %(description)s,X ordernum %(ordernum)s,X relation_type ER,X from_entity SE,X to_entity OE WHERE SE name %(se)s,ER name %(rt)s,OE name %(oe)s',
-             {'rt': 'add_permission', 'description': u'groups allowed to add entities/relations of this type', 'composite': None, 'oe': 'CWGroup', 'ordernum': 9999, 'cardinality': u'**', 'se': 'CWRelation'}),
-            ('INSERT CWRelation X: X cardinality %(cardinality)s,X composite %(composite)s,X description %(description)s,X ordernum %(ordernum)s,X relation_type ER,X from_entity SE,X to_entity OE WHERE SE name %(se)s,ER name %(rt)s,OE name %(oe)s',
-             {'rt': 'add_permission', 'description': u'rql expression allowing to add entities/relations of this type', 'composite': 'subject', 'oe': 'RQLExpression', 'ordernum': 9999, 'cardinality': u'*?', 'se': 'CWRelation'}),
+            ('INSERT CWRelation X: X cardinality %(cardinality)s,X composite %(composite)s,X description %(description)s,X ordernum %(ordernum)s,X relation_type ER,X from_entity SE,X to_entity OE WHERE SE eid %(se)s,ER eid %(rt)s,OE eid %(oe)s',
+             {'se': None, 'rt': None, 'oe': None,
+              'description': u'groups allowed to add entities/relations of this type', 'composite': None, 'ordernum': 9999, 'cardinality': u'**'}),
+            ('INSERT CWRelation X: X cardinality %(cardinality)s,X composite %(composite)s,X description %(description)s,X ordernum %(ordernum)s,X relation_type ER,X from_entity SE,X to_entity OE WHERE SE eid %(se)s,ER eid %(rt)s,OE eid %(oe)s',
+             {'se': None, 'rt': None, 'oe': None,
+              'description': u'rql expression allowing to add entities/relations of this type', 'composite': 'subject', 'ordernum': 9999, 'cardinality': u'*?'}),
             ])
 
     def test_rschema2rql3(self):
-        self.assertListEquals(list(rschema2rql(schema.rschema('cardinality'))),
+        self.assertListEquals(list(rschema2rql(schema.rschema('cardinality'), cstrtypemap)),
                              [
             ('INSERT CWRType X: X description %(description)s,X final %(final)s,X fulltext_container %(fulltext_container)s,X inlined %(inlined)s,X name %(name)s,X symmetric %(symmetric)s',
              {'description': u'', 'symmetric': False, 'name': u'cardinality', 'final': True, 'fulltext_container': None, 'inlined': False}),
 
-            ('INSERT CWAttribute X: X cardinality %(cardinality)s,X defaultval %(defaultval)s,X description %(description)s,X fulltextindexed %(fulltextindexed)s,X indexed %(indexed)s,X internationalizable %(internationalizable)s,X ordernum %(ordernum)s,X relation_type ER,X from_entity SE,X to_entity OE WHERE SE name %(se)s,ER name %(rt)s,OE name %(oe)s',
-             {'rt': 'cardinality', 'description': u'subject/object cardinality', 'internationalizable': True, 'fulltextindexed': False, 'ordernum': 5, 'defaultval': None, 'indexed': False, 'cardinality': u'?1', 'oe': 'String', 'se': 'CWAttribute'}),
-            ('INSERT CWConstraint X: X value %(value)s, X cstrtype CT, EDEF constrained_by X WHERE CT name %(ctname)s, EDEF relation_type ER, EDEF from_entity SE, EDEF to_entity OE, ER name %(rt)s, SE name %(se)s, OE name %(oe)s, EDEF is CWAttribute',
-             {'rt': 'cardinality', 'oe': 'String', 'ctname': u'SizeConstraint', 'se': 'CWAttribute', 'value': u'max=2'}),
-            ('INSERT CWConstraint X: X value %(value)s, X cstrtype CT, EDEF constrained_by X WHERE CT name %(ctname)s, EDEF relation_type ER, EDEF from_entity SE, EDEF to_entity OE, ER name %(rt)s, SE name %(se)s, OE name %(oe)s, EDEF is CWAttribute',
-             {'rt': 'cardinality', 'oe': 'String', 'ctname': u'StaticVocabularyConstraint', 'se': 'CWAttribute', 'value': u"u'?1', u'11'"}),
+            ('INSERT CWAttribute X: X cardinality %(cardinality)s,X defaultval %(defaultval)s,X description %(description)s,X fulltextindexed %(fulltextindexed)s,X indexed %(indexed)s,X internationalizable %(internationalizable)s,X ordernum %(ordernum)s,X relation_type ER,X from_entity SE,X to_entity OE WHERE SE eid %(se)s,ER eid %(rt)s,OE eid %(oe)s',
+             {'se': None, 'rt': None, 'oe': None,
+              'description': u'subject/object cardinality', 'internationalizable': True, 'fulltextindexed': False, 'ordernum': 5, 'defaultval': None, 'indexed': False, 'cardinality': u'?1'}),
+            ('INSERT CWConstraint X: X value %(value)s, X cstrtype CT, EDEF constrained_by X WHERE CT eid %(ct)s, EDEF eid %(x)s',
+             {'x': None, 'ct': u'SizeConstraint_eid', 'value': u'max=2'}),
+            ('INSERT CWConstraint X: X value %(value)s, X cstrtype CT, EDEF constrained_by X WHERE CT eid %(ct)s, EDEF eid %(x)s',
+             {'x': None, 'ct': u'StaticVocabularyConstraint_eid', 'value': u"u'?1', u'11'"}),
 
-            ('INSERT CWAttribute X: X cardinality %(cardinality)s,X defaultval %(defaultval)s,X description %(description)s,X fulltextindexed %(fulltextindexed)s,X indexed %(indexed)s,X internationalizable %(internationalizable)s,X ordernum %(ordernum)s,X relation_type ER,X from_entity SE,X to_entity OE WHERE SE name %(se)s,ER name %(rt)s,OE name %(oe)s',
-             {'rt': 'cardinality', 'description': u'subject/object cardinality', 'internationalizable': True, 'fulltextindexed': False, 'ordernum': 5, 'defaultval': None, 'indexed': False, 'cardinality': u'?1', 'oe': 'String', 'se': 'CWRelation'}),
-            ('INSERT CWConstraint X: X value %(value)s, X cstrtype CT, EDEF constrained_by X WHERE CT name %(ctname)s, EDEF relation_type ER, EDEF from_entity SE, EDEF to_entity OE, ER name %(rt)s, SE name %(se)s, OE name %(oe)s, EDEF is CWAttribute',
-             {'rt': 'cardinality', 'oe': 'String', 'ctname': u'SizeConstraint', 'se': 'CWRelation', 'value': u'max=2'}),
-            ('INSERT CWConstraint X: X value %(value)s, X cstrtype CT, EDEF constrained_by X WHERE CT name %(ctname)s, EDEF relation_type ER, EDEF from_entity SE, EDEF to_entity OE, ER name %(rt)s, SE name %(se)s, OE name %(oe)s, EDEF is CWAttribute',
-             {'rt': 'cardinality', 'oe': 'String', 'ctname': u'StaticVocabularyConstraint', 'se': 'CWRelation', 'value': u"u'?*', u'1*', u'+*', u'**', u'?+', u'1+', u'++', u'*+', u'?1', u'11', u'+1', u'*1', u'??', u'1?', u'+?', u'*?'"}),
+            ('INSERT CWAttribute X: X cardinality %(cardinality)s,X defaultval %(defaultval)s,X description %(description)s,X fulltextindexed %(fulltextindexed)s,X indexed %(indexed)s,X internationalizable %(internationalizable)s,X ordernum %(ordernum)s,X relation_type ER,X from_entity SE,X to_entity OE WHERE SE eid %(se)s,ER eid %(rt)s,OE eid %(oe)s',
+             {'se': None, 'rt': None, 'oe': None,
+              'description': u'subject/object cardinality', 'internationalizable': True, 'fulltextindexed': False, 'ordernum': 5, 'defaultval': None, 'indexed': False, 'cardinality': u'?1'}),
+            ('INSERT CWConstraint X: X value %(value)s, X cstrtype CT, EDEF constrained_by X WHERE CT eid %(ct)s, EDEF eid %(x)s',
+             {'x': None, 'ct': u'SizeConstraint_eid', 'value': u'max=2'}),
+            ('INSERT CWConstraint X: X value %(value)s, X cstrtype CT, EDEF constrained_by X WHERE CT eid %(ct)s, EDEF eid %(x)s',
+             {'x': None, 'ct': u'StaticVocabularyConstraint_eid', 'value': u"u'?*', u'1*', u'+*', u'**', u'?+', u'1+', u'++', u'*+', u'?1', u'11', u'+1', u'*1', u'??', u'1?', u'+?', u'*?'"}),
             ])
 
+    def test_rdef2rql(self):
+        self.assertListEquals(list(rdef2rql(schema['description_format'].rdefs[('CWRType', 'String')], cstrtypemap)),
+                              [
+            ('INSERT CWAttribute X: X cardinality %(cardinality)s,X defaultval %(defaultval)s,X description %(description)s,X fulltextindexed %(fulltextindexed)s,X indexed %(indexed)s,X internationalizable %(internationalizable)s,X ordernum %(ordernum)s,X relation_type ER,X from_entity SE,X to_entity OE WHERE SE eid %(se)s,ER eid %(rt)s,OE eid %(oe)s',
+             {'se': None, 'rt': None, 'oe': None,
+              'description': u'', 'internationalizable': True, 'fulltextindexed': False, 'ordernum': 7, 'defaultval': u'text/plain', 'indexed': False, 'cardinality': u'?1'}),
+            ('INSERT CWConstraint X: X value %(value)s, X cstrtype CT, EDEF constrained_by X WHERE CT eid %(ct)s, EDEF eid %(x)s',
+             {'x': None, 'value': u'None', 'ct': 'FormatConstraint_eid'}),
+            ('INSERT CWConstraint X: X value %(value)s, X cstrtype CT, EDEF constrained_by X WHERE CT eid %(ct)s, EDEF eid %(x)s',
+             {'x': None, 'value': u'max=50', 'ct': 'SizeConstraint_eid'})])
+
 
     def test_updateeschema2rql1(self):
-        self.assertListEquals(list(updateeschema2rql(schema.eschema('CWAttribute'))),
-                              [('SET X description %(description)s,X final %(final)s,X name %(name)s WHERE X is CWEType, X name %(et)s',
-                                {'description': u'define a final relation: link a final relation type from a non final entity to a final entity type. used to build the instance schema', 'et': 'CWAttribute', 'final': False, 'name': u'CWAttribute'}),
+        self.assertListEquals(list(updateeschema2rql(schema.eschema('CWAttribute'), 1)),
+                              [('SET X description %(description)s,X final %(final)s,X name %(name)s WHERE X eid %(x)s',
+                                {'description': u'define a final relation: link a final relation type from a non final entity to a final entity type. used to build the instance schema', 'x': 1, 'final': False, 'name': u'CWAttribute'}),
                                ])
 
     def test_updateeschema2rql2(self):
-        self.assertListEquals(list(updateeschema2rql(schema.eschema('String'))),
-                              [('SET X description %(description)s,X final %(final)s,X name %(name)s WHERE X is CWEType, X name %(et)s',
-                                {'description': u'', 'et': 'String', 'final': True, 'name': u'String'})
+        self.assertListEquals(list(updateeschema2rql(schema.eschema('String'), 1)),
+                              [('SET X description %(description)s,X final %(final)s,X name %(name)s WHERE X eid %(x)s',
+                                {'description': u'', 'x': 1, 'final': True, 'name': u'String'})
                                ])
 
     def test_updaterschema2rql1(self):
-        self.assertListEquals(list(updaterschema2rql(schema.rschema('relation_type'))),
+        self.assertListEquals(list(updaterschema2rql(schema.rschema('relation_type'), 1)),
                              [
-            ('SET X description %(description)s,X final %(final)s,X fulltext_container %(fulltext_container)s,X inlined %(inlined)s,X name %(name)s,X symmetric %(symmetric)s WHERE X is CWRType, X name %(rt)s',
-             {'rt': 'relation_type', 'symmetric': False,
+            ('SET X description %(description)s,X final %(final)s,X fulltext_container %(fulltext_container)s,X inlined %(inlined)s,X name %(name)s,X symmetric %(symmetric)s WHERE X eid %(x)s',
+             {'x': 1, 'symmetric': False,
               'description': u'link a relation definition to its relation type',
               'final': False, 'fulltext_container': None, 'inlined': True, 'name': u'relation_type'})
             ])
 
     def test_updaterschema2rql2(self):
         expected = [
-            ('SET X description %(description)s,X final %(final)s,X fulltext_container %(fulltext_container)s,X inlined %(inlined)s,X name %(name)s,X symmetric %(symmetric)s WHERE X is CWRType, X name %(rt)s',
-             {'rt': 'add_permission', 'symmetric': False,
+            ('SET X description %(description)s,X final %(final)s,X fulltext_container %(fulltext_container)s,X inlined %(inlined)s,X name %(name)s,X symmetric %(symmetric)s WHERE X eid %(x)s',
+             {'x': 1, 'symmetric': False,
               'description': u'', 'final': False, 'fulltext_container': None,
               'inlined': False, 'name': u'add_permission'})
             ]
-        for i, (rql, args) in enumerate(updaterschema2rql(schema.rschema('add_permission'))):
+        for i, (rql, args) in enumerate(updaterschema2rql(schema.rschema('add_permission'), 1)):
             yield self.assertEquals, (rql, args), expected[i]
 
 class Perms2RQLTC(TestCase):
@@ -144,35 +170,29 @@
 
     def test_eperms2rql1(self):
         self.assertListEquals([(rql, kwargs) for rql, kwargs in erperms2rql(schema.eschema('CWEType'), self.GROUP_MAPPING)],
-                              [('SET X read_permission Y WHERE Y eid %(g)s, ', {'g': 0}),
-                               ('SET X read_permission Y WHERE Y eid %(g)s, ', {'g': 1}),
-                               ('SET X read_permission Y WHERE Y eid %(g)s, ', {'g': 2}),
-                               ('SET X add_permission Y WHERE Y eid %(g)s, ', {'g': 0}),
-                               ('SET X update_permission Y WHERE Y eid %(g)s, ', {'g': 0}),
-                               ('SET X update_permission Y WHERE Y eid %(g)s, ', {'g': 3}),
-                               ('SET X delete_permission Y WHERE Y eid %(g)s, ', {'g': 0}),
+                              [('SET X read_permission Y WHERE Y eid %(g)s, X eid %(x)s', {'g': 0}),
+                               ('SET X read_permission Y WHERE Y eid %(g)s, X eid %(x)s', {'g': 1}),
+                               ('SET X read_permission Y WHERE Y eid %(g)s, X eid %(x)s', {'g': 2}),
+                               ('SET X add_permission Y WHERE Y eid %(g)s, X eid %(x)s', {'g': 0}),
+                               ('SET X update_permission Y WHERE Y eid %(g)s, X eid %(x)s', {'g': 0}),
+                               ('SET X delete_permission Y WHERE Y eid %(g)s, X eid %(x)s', {'g': 0}),
                                ])
 
     def test_rperms2rql2(self):
         self.assertListEquals([(rql, kwargs) for rql, kwargs in erperms2rql(schema.rschema('read_permission').rdef('CWEType', 'CWGroup'), self.GROUP_MAPPING)],
-                              [('SET X read_permission Y WHERE Y eid %(g)s, ', {'g': 0}),
-                               ('SET X read_permission Y WHERE Y eid %(g)s, ', {'g': 1}),
-                               ('SET X read_permission Y WHERE Y eid %(g)s, ', {'g': 2}),
-                               ('SET X add_permission Y WHERE Y eid %(g)s, ', {'g': 0}),
-                               ('SET X delete_permission Y WHERE Y eid %(g)s, ', {'g': 0}),
+                              [('SET X read_permission Y WHERE Y eid %(g)s, X eid %(x)s', {'g': 0}),
+                               ('SET X read_permission Y WHERE Y eid %(g)s, X eid %(x)s', {'g': 1}),
+                               ('SET X read_permission Y WHERE Y eid %(g)s, X eid %(x)s', {'g': 2}),
+                               ('SET X add_permission Y WHERE Y eid %(g)s, X eid %(x)s', {'g': 0}),
+                               ('SET X delete_permission Y WHERE Y eid %(g)s, X eid %(x)s', {'g': 0}),
                                ])
 
     def test_rperms2rql3(self):
         self.assertListEquals([(rql, kwargs) for rql, kwargs in erperms2rql(schema.rschema('name').rdef('CWEType', 'String'), self.GROUP_MAPPING)],
-                              [('SET X read_permission Y WHERE Y eid %(g)s, ', {'g': 0}),
-                               ('SET X read_permission Y WHERE Y eid %(g)s, ', {'g': 1}),
-                               ('SET X read_permission Y WHERE Y eid %(g)s, ', {'g': 2}),
-                               ('SET X update_permission Y WHERE Y eid %(g)s, ', {'g': 0}),
-                               ('INSERT RQLExpression E: '
-                                'E expression %(e)s, E exprtype %(t)s, E mainvars %(v)s, '
-                                'X update_permission E '
-                                'WHERE ', # completed by the outer function
-                                {'e': u'U has_update_permission X', 't': u'ERQLExpression', 'v': u'X'}),
+                              [('SET X read_permission Y WHERE Y eid %(g)s, X eid %(x)s', {'g': 0}),
+                               ('SET X read_permission Y WHERE Y eid %(g)s, X eid %(x)s', {'g': 1}),
+                               ('SET X read_permission Y WHERE Y eid %(g)s, X eid %(x)s', {'g': 2}),
+                               ('SET X update_permission Y WHERE Y eid %(g)s, X eid %(x)s', {'g': 0}),
                                ])
 
     #def test_perms2rql(self):