server/schemaserial.py
changeset 9490 b3d2c4065e6a
parent 9489 0581f6d2812e
child 9503 562cef272a35
equal deleted inserted replaced
9489:0581f6d2812e 9490:b3d2c4065e6a
    79     map = dict(cnx.execute('Any T, X WHERE X is CWConstraintType, X name T'))
    79     map = dict(cnx.execute('Any T, X WHERE X is CWConstraintType, X name T'))
    80     return map
    80     return map
    81 
    81 
    82 # schema / perms deserialization ##############################################
    82 # schema / perms deserialization ##############################################
    83 
    83 
    84 def deserialize_schema(schema, session):
    84 def deserialize_schema(schema, cnx):
    85     """return a schema according to information stored in an rql database
    85     """return a schema according to information stored in an rql database
    86     as CWRType and CWEType entities
    86     as CWRType and CWEType entities
    87     """
    87     """
    88     repo = session.repo
    88     repo = cnx.repo
    89     dbhelper = repo.system_source.dbhelper
    89     dbhelper = repo.system_source.dbhelper
    90     # XXX bw compat (3.6 migration)
    90     # XXX bw compat (3.6 migration)
    91     sqlcu = session.cnxset.cu
    91     with cnx.ensure_cnx_set:
    92     sqlcu.execute("SELECT * FROM cw_CWRType WHERE cw_name='symetric'")
    92         sqlcu = cnx.system_sql("SELECT * FROM cw_CWRType WHERE cw_name='symetric'")
    93     if sqlcu.fetchall():
    93         if sqlcu.fetchall():
    94         sql = dbhelper.sql_rename_col('cw_CWRType', 'cw_symetric', 'cw_symmetric',
    94             sql = dbhelper.sql_rename_col('cw_CWRType', 'cw_symetric', 'cw_symmetric',
    95                                       dbhelper.TYPE_MAPPING['Boolean'], True)
    95                                           dbhelper.TYPE_MAPPING['Boolean'], True)
    96         sqlcu.execute(sql)
    96             sqlcu.execute(sql)
    97         sqlcu.execute("UPDATE cw_CWRType SET cw_name='symmetric' WHERE cw_name='symetric'")
    97             sqlcu.execute("UPDATE cw_CWRType SET cw_name='symmetric' WHERE cw_name='symetric'")
    98         session.commit(False)
    98             cnx.commit(False)
    99     ertidx = {}
    99     ertidx = {}
   100     copiedeids = set()
   100     copiedeids = set()
   101     permsidx = deserialize_ertype_permissions(session)
   101     permsidx = deserialize_ertype_permissions(cnx)
   102     schema.reading_from_database = True
   102     schema.reading_from_database = True
   103     for eid, etype, desc in session.execute(
   103     for eid, etype, desc in cnx.execute(
   104         'Any X, N, D WHERE X is CWEType, X name N, X description D',
   104         'Any X, N, D WHERE X is CWEType, X name N, X description D',
   105         build_descr=False):
   105         build_descr=False):
   106         # base types are already in the schema, skip them
   106         # base types are already in the schema, skip them
   107         if etype in schemamod.BASE_TYPES:
   107         if etype in schemamod.BASE_TYPES:
   108             # just set the eid
   108             # just set the eid
   112             continue
   112             continue
   113         if etype in ETYPE_NAME_MAP:
   113         if etype in ETYPE_NAME_MAP:
   114             needcopy = False
   114             needcopy = False
   115             netype = ETYPE_NAME_MAP[etype]
   115             netype = ETYPE_NAME_MAP[etype]
   116             # can't use write rql queries at this point, use raw sql
   116             # can't use write rql queries at this point, use raw sql
   117             sqlexec = session.system_sql
   117             sqlexec = cnx.system_sql
   118             if sqlexec('SELECT 1 FROM %(p)sCWEType WHERE %(p)sname=%%(n)s'
   118             if sqlexec('SELECT 1 FROM %(p)sCWEType WHERE %(p)sname=%%(n)s'
   119                        % {'p': sqlutils.SQL_PREFIX}, {'n': netype}).fetchone():
   119                        % {'p': sqlutils.SQL_PREFIX}, {'n': netype}).fetchone():
   120                 # the new type already exists, we should copy (eg make existing
   120                 # the new type already exists, we should copy (eg make existing
   121                 # instances of the old type instances of the new type)
   121                 # instances of the old type instances of the new type)
   122                 assert etype.lower() != netype.lower()
   122                 assert etype.lower() != netype.lower()
   129                     alter_table_sql = dbhelper.sql_rename_table(sqlutils.SQL_PREFIX+etype,
   129                     alter_table_sql = dbhelper.sql_rename_table(sqlutils.SQL_PREFIX+etype,
   130                                                                 sqlutils.SQL_PREFIX+netype)
   130                                                                 sqlutils.SQL_PREFIX+netype)
   131                     sqlexec(alter_table_sql)
   131                     sqlexec(alter_table_sql)
   132             sqlexec('UPDATE entities SET type=%(n)s WHERE type=%(x)s',
   132             sqlexec('UPDATE entities SET type=%(n)s WHERE type=%(x)s',
   133                     {'x': etype, 'n': netype})
   133                     {'x': etype, 'n': netype})
   134             session.commit(False)
   134             cnx.commit(False)
   135             tocleanup = [eid]
   135             tocleanup = [eid]
   136             tocleanup += (eid for eid, cached in repo._type_source_cache.iteritems()
   136             tocleanup += (eid for eid, cached in repo._type_source_cache.iteritems()
   137                           if etype == cached[0])
   137                           if etype == cached[0])
   138             repo.clear_caches(tocleanup)
   138             repo.clear_caches(tocleanup)
   139             session.commit(False)
   139             cnx.commit(False)
   140             if needcopy:
   140             if needcopy:
   141                 ertidx[eid] = netype
   141                 ertidx[eid] = netype
   142                 copiedeids.add(eid)
   142                 copiedeids.add(eid)
   143                 # copy / CWEType entity removal expected to be done through
   143                 # copy / CWEType entity removal expected to be done through
   144                 # rename_entity_type in a migration script
   144                 # rename_entity_type in a migration script
   146             etype = netype
   146             etype = netype
   147         ertidx[eid] = etype
   147         ertidx[eid] = etype
   148         eschema = schema.add_entity_type(
   148         eschema = schema.add_entity_type(
   149             ybo.EntityType(name=etype, description=desc, eid=eid))
   149             ybo.EntityType(name=etype, description=desc, eid=eid))
   150         set_perms(eschema, permsidx)
   150         set_perms(eschema, permsidx)
   151     for etype, stype in session.execute(
   151     for etype, stype in cnx.execute(
   152         'Any XN, ETN WHERE X is CWEType, X name XN, X specializes ET, ET name ETN',
   152         'Any XN, ETN WHERE X is CWEType, X name XN, X specializes ET, ET name ETN',
   153         build_descr=False):
   153         build_descr=False):
   154         etype = ETYPE_NAME_MAP.get(etype, etype)
   154         etype = ETYPE_NAME_MAP.get(etype, etype)
   155         stype = ETYPE_NAME_MAP.get(stype, stype)
   155         stype = ETYPE_NAME_MAP.get(stype, stype)
   156         schema.eschema(etype)._specialized_type = stype
   156         schema.eschema(etype)._specialized_type = stype
   157         schema.eschema(stype)._specialized_by.append(etype)
   157         schema.eschema(stype)._specialized_by.append(etype)
   158     for eid, rtype, desc, sym, il, ftc in session.execute(
   158     for eid, rtype, desc, sym, il, ftc in cnx.execute(
   159         'Any X,N,D,S,I,FTC WHERE X is CWRType, X name N, X description D, '
   159         'Any X,N,D,S,I,FTC WHERE X is CWRType, X name N, X description D, '
   160         'X symmetric S, X inlined I, X fulltext_container FTC', build_descr=False):
   160         'X symmetric S, X inlined I, X fulltext_container FTC', build_descr=False):
   161         ertidx[eid] = rtype
   161         ertidx[eid] = rtype
   162         rschema = schema.add_relation_type(
   162         rschema = schema.add_relation_type(
   163             ybo.RelationType(name=rtype, description=desc,
   163             ybo.RelationType(name=rtype, description=desc,
   164                              symmetric=bool(sym), inlined=bool(il),
   164                              symmetric=bool(sym), inlined=bool(il),
   165                              fulltext_container=ftc, eid=eid))
   165                              fulltext_container=ftc, eid=eid))
   166     cstrsidx = deserialize_rdef_constraints(session)
   166     cstrsidx = deserialize_rdef_constraints(cnx)
   167     pendingrdefs = []
   167     pendingrdefs = []
   168     # closure to factorize common code of attribute/relation rdef addition
   168     # closure to factorize common code of attribute/relation rdef addition
   169     def _add_rdef(rdefeid, seid, reid, oeid, **kwargs):
   169     def _add_rdef(rdefeid, seid, reid, oeid, **kwargs):
   170         rdef = ybo.RelationDefinition(ertidx[seid], ertidx[reid], ertidx[oeid],
   170         rdef = ybo.RelationDefinition(ertidx[seid], ertidx[reid], ertidx[oeid],
   171                                       constraints=cstrsidx.get(rdefeid, ()),
   171                                       constraints=cstrsidx.get(rdefeid, ()),
   190             if rdefs is not None:
   190             if rdefs is not None:
   191                 ertidx[rdefeid] = rdefs
   191                 ertidx[rdefeid] = rdefs
   192                 set_perms(rdefs, permsidx)
   192                 set_perms(rdefs, permsidx)
   193     # Get the type parameters for additional base types.
   193     # Get the type parameters for additional base types.
   194     try:
   194     try:
   195         extra_props = dict(session.execute('Any X, XTP WHERE X is CWAttribute, '
   195         extra_props = dict(cnx.execute('Any X, XTP WHERE X is CWAttribute, '
   196                                            'X extra_props XTP'))
   196                                            'X extra_props XTP'))
   197     except Exception:
   197     except Exception:
   198         session.critical('Previous CRITICAL notification about extra_props is not '
   198         cnx.critical('Previous CRITICAL notification about extra_props is not '
   199                          'a problem if you are migrating to cubicweb 3.17')
   199                          'a problem if you are migrating to cubicweb 3.17')
   200         extra_props = {} # not yet in the schema (introduced by 3.17 migration)
   200         extra_props = {} # not yet in the schema (introduced by 3.17 migration)
   201     for values in session.execute(
   201     for values in cnx.execute(
   202         'Any X,SE,RT,OE,CARD,ORD,DESC,IDX,FTIDX,I18N,DFLT WHERE X is CWAttribute,'
   202         'Any X,SE,RT,OE,CARD,ORD,DESC,IDX,FTIDX,I18N,DFLT WHERE X is CWAttribute,'
   203         'X relation_type RT, X cardinality CARD, X ordernum ORD, X indexed IDX,'
   203         'X relation_type RT, X cardinality CARD, X ordernum ORD, X indexed IDX,'
   204         'X description DESC, X internationalizable I18N, X defaultval DFLT,'
   204         'X description DESC, X internationalizable I18N, X defaultval DFLT,'
   205         'X fulltextindexed FTIDX, X from_entity SE, X to_entity OE',
   205         'X fulltextindexed FTIDX, X from_entity SE, X to_entity OE',
   206         build_descr=False):
   206         build_descr=False):
   214                 default = default.unzpickle()
   214                 default = default.unzpickle()
   215         _add_rdef(rdefeid, seid, reid, oeid,
   215         _add_rdef(rdefeid, seid, reid, oeid,
   216                   cardinality=card, description=desc, order=ord,
   216                   cardinality=card, description=desc, order=ord,
   217                   indexed=idx, fulltextindexed=ftidx, internationalizable=i18n,
   217                   indexed=idx, fulltextindexed=ftidx, internationalizable=i18n,
   218                   default=default, **typeparams)
   218                   default=default, **typeparams)
   219     for values in session.execute(
   219     for values in cnx.execute(
   220         'Any X,SE,RT,OE,CARD,ORD,DESC,C WHERE X is CWRelation, X relation_type RT,'
   220         'Any X,SE,RT,OE,CARD,ORD,DESC,C WHERE X is CWRelation, X relation_type RT,'
   221         'X cardinality CARD, X ordernum ORD, X description DESC, '
   221         'X cardinality CARD, X ordernum ORD, X description DESC, '
   222         'X from_entity SE, X to_entity OE, X composite C', build_descr=False):
   222         'X from_entity SE, X to_entity OE, X composite C', build_descr=False):
   223         rdefeid, seid, reid, oeid, card, ord, desc, comp = values
   223         rdefeid, seid, reid, oeid, card, ord, desc, comp = values
   224         _add_rdef(rdefeid, seid, reid, oeid,
   224         _add_rdef(rdefeid, seid, reid, oeid,
   230         except BadSchemaDefinition:
   230         except BadSchemaDefinition:
   231             continue
   231             continue
   232         if rdefs is not None:
   232         if rdefs is not None:
   233             set_perms(rdefs, permsidx)
   233             set_perms(rdefs, permsidx)
   234     unique_togethers = {}
   234     unique_togethers = {}
   235     rset = session.execute(
   235     rset = cnx.execute(
   236     'Any X,E,R WHERE '
   236     'Any X,E,R WHERE '
   237     'X is CWUniqueTogetherConstraint, '
   237     'X is CWUniqueTogetherConstraint, '
   238     'X constraint_of E, X relations R', build_descr=False)
   238     'X constraint_of E, X relations R', build_descr=False)
   239     for values in rset:
   239     for values in rset:
   240         uniquecstreid, eeid, releid = values
   240         uniquecstreid, eeid, releid = values
   249             rtype = str(rel)
   249             rtype = str(rel)
   250         relations[1].append(rtype)
   250         relations[1].append(rtype)
   251     for eschema, unique_together in unique_togethers.itervalues():
   251     for eschema, unique_together in unique_togethers.itervalues():
   252         eschema._unique_together.append(tuple(sorted(unique_together)))
   252         eschema._unique_together.append(tuple(sorted(unique_together)))
   253     schema.infer_specialization_rules()
   253     schema.infer_specialization_rules()
   254     session.commit()
   254     cnx.commit()
   255     schema.reading_from_database = False
   255     schema.reading_from_database = False
   256 
   256 
   257 
   257 
   258 def deserialize_ertype_permissions(session):
   258 def deserialize_ertype_permissions(cnx):
   259     """return sect action:groups associations for the given
   259     """return sect action:groups associations for the given
   260     entity or relation schema with its eid, according to schema's
   260     entity or relation schema with its eid, according to schema's
   261     permissions stored in the database as [read|add|delete|update]_permission
   261     permissions stored in the database as [read|add|delete|update]_permission
   262     relations between CWEType/CWRType and CWGroup entities
   262     relations between CWEType/CWRType and CWGroup entities
   263     """
   263     """
   264     res = {}
   264     res = {}
   265     for action in ('read', 'add', 'update', 'delete'):
   265     for action in ('read', 'add', 'update', 'delete'):
   266         rql = 'Any E,N WHERE G is CWGroup, G name N, E %s_permission G' % action
   266         rql = 'Any E,N WHERE G is CWGroup, G name N, E %s_permission G' % action
   267         for eid, gname in session.execute(rql, build_descr=False):
   267         for eid, gname in cnx.execute(rql, build_descr=False):
   268             res.setdefault(eid, {}).setdefault(action, []).append(gname)
   268             res.setdefault(eid, {}).setdefault(action, []).append(gname)
   269         rql = ('Any E,X,EXPR,V WHERE X is RQLExpression, X expression EXPR, '
   269         rql = ('Any E,X,EXPR,V WHERE X is RQLExpression, X expression EXPR, '
   270                'E %s_permission X, X mainvars V' % action)
   270                'E %s_permission X, X mainvars V' % action)
   271         for eid, expreid, expr, mainvars in session.execute(rql, build_descr=False):
   271         for eid, expreid, expr, mainvars in cnx.execute(rql, build_descr=False):
   272             # we don't know yet if it's a rql expr for an entity or a relation,
   272             # we don't know yet if it's a rql expr for an entity or a relation,
   273             # so append a tuple to differentiate from groups and so we'll be
   273             # so append a tuple to differentiate from groups and so we'll be
   274             # able to instantiate it later
   274             # able to instantiate it later
   275             res.setdefault(eid, {}).setdefault(action, []).append( (expr, mainvars, expreid) )
   275             res.setdefault(eid, {}).setdefault(action, []).append( (expr, mainvars, expreid) )
   276     return res
   276     return res
   277 
   277 
   278 def deserialize_rdef_constraints(session):
   278 def deserialize_rdef_constraints(cnx):
   279     """return the list of relation definition's constraints as instances"""
   279     """return the list of relation definition's constraints as instances"""
   280     res = {}
   280     res = {}
   281     for rdefeid, ceid, ct, val in session.execute(
   281     for rdefeid, ceid, ct, val in cnx.execute(
   282         'Any E, X,TN,V WHERE E constrained_by X, X is CWConstraint, '
   282         'Any E, X,TN,V WHERE E constrained_by X, X is CWConstraint, '
   283         'X cstrtype T, T name TN, X value V', build_descr=False):
   283         'X cstrtype T, T name TN, X value V', build_descr=False):
   284         cstr = CONSTRAINTS[ct].deserialize(val)
   284         cstr = CONSTRAINTS[ct].deserialize(val)
   285         cstr.eid = ceid
   285         cstr.eid = ceid
   286         res.setdefault(rdefeid, []).append(cstr)
   286         res.setdefault(rdefeid, []).append(cstr)