1 """functions for schema / permissions (de)serialization using RQL

     2 

     3 :organization: Logilab

     4 :copyright: 2001-2008 LOGILAB S.A. (Paris, FRANCE), all rights reserved.

     5 :contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr

     6 """

     7 __docformat__ = "restructuredtext en"

     8 

     9 from itertools import chain

    10 

    11 from logilab.common.shellutils import ProgressBar

    12 

    13 from yams import schema as schemamod, buildobjs as ybo

    14 

    15 from cubicweb.schema import CONSTRAINTS, ETYPE_NAME_MAP

    16 

    17 def group_mapping(cursor, interactive=True):

    18     """create a group mapping from an rql cursor

    19 

    20     A group mapping has standard group names as key (managers, owners at least)

    21     and the actual EGroup entity's eid as associated value.

    22     In interactive mode (the default), missing groups'eid will be prompted

    23     from the user.

    24     """

    25     res = {}

    26     for eid, name in cursor.execute('Any G, N WHERE G is EGroup, G name N'):

    27         res[name] = eid

    28     if not interactive:

    29         return res

    30     missing = [g for g in ('owners', 'managers', 'users', 'guests') if not g in res]

    31     if missing:

    32         print 'some native groups are missing but the following groups have been found:'

    33         print '\n'.join('* %s (%s)' % (n, eid) for n, eid in res.items())

    34         print 

    35         print 'enter the eid of a to group to map to each missing native group'

    36         print 'or just type enter to skip permissions granted to a group'

    37         for group in missing:

    38             while True:

    39                 value = raw_input('eid for group %s: ' % group).strip()

    40                 if not value:

    41                     continue

    42                 try:

    43                     res[group] = int(value)

    44                 except ValueError:

    45                     print 'eid should be an integer'

    46                     continue

    47     return res

    48 

    49 # schema / perms deserialization ##############################################

    50 

    51 def deserialize_schema(schema, session):

    52     """return a schema according to information stored in an rql database

    53     as ERType and EEType entities

    54     """

    55     # print 'reading schema from the database...'

    56     index = {}

    57     permsdict = deserialize_ertype_permissions(session)

    58     schema.reading_from_database = True

    59     for eid, etype, desc, meta in session.execute('Any X, N, D, M WHERE '

    60                                                   'X is EEType, X name N, '

    61                                                   'X description D, X meta M',

    62                                                   build_descr=False):

    63         # base types are already in the schema, skip them

    64         if etype in schemamod.BASE_TYPES:

    65             # just set the eid

    66             eschema = schema.eschema(etype)

    67             eschema.eid = eid

    68             index[eid] = eschema

    69             continue

    70         if etype in ETYPE_NAME_MAP: # XXX <2.45 bw compat

    71             print 'fixing etype name from %s to %s' % (etype, ETYPE_NAME_MAP[etype])

    72             # can't use write rql queries at this point, use raw sql

    73             session.system_sql('UPDATE EEType SET name=%(n)s WHERE eid=%(x)s',

    74                                {'x': eid, 'n': ETYPE_NAME_MAP[etype]})

    75             session.system_sql('UPDATE entities SET type=%(n)s WHERE type=%(x)s',

    76                                {'x': etype, 'n': ETYPE_NAME_MAP[etype]})

    77             session.commit(False)

    78             try:

    79                 session.system_sql('UPDATE deleted_entities SET type=%(n)s WHERE type=%(x)s',

    80                                    {'x': etype, 'n': ETYPE_NAME_MAP[etype]})

    81             except:

    82                 pass

    83             tocleanup = [eid]

    84             tocleanup += (eid for eid, (eidetype, uri, extid) in session.repo._type_source_cache.items()

    85                           if etype == eidetype)

    86             session.repo.clear_caches(tocleanup)

    87             session.commit(False)

    88             etype = ETYPE_NAME_MAP[etype]

    89         etype = ybo.EntityType(name=etype, description=desc, meta=meta, eid=eid)

    90         eschema = schema.add_entity_type(etype)

    91         index[eid] = eschema

    92         set_perms(eschema, permsdict.get(eid, {}))

    93     try:

    94         rset = session.execute('Any XN, ETN WHERE X is EEType, X name XN, '

    95                                'X specializes ET, ET name ETN')

    96     except: # `specializes` relation not available for versions prior to 2.50

    97         session.rollback(False)

    98     else:

    99         for etype, stype in rset:

   100             eschema = schema.eschema(etype)

   101             seschema = schema.eschema(stype)

   102             eschema._specialized_type = stype

   103             seschema._specialized_by.append(etype)

   104     for eid, rtype, desc, meta, sym, il in session.execute(

   105         'Any X,N,D,M,S,I WHERE X is ERType, X name N, X description D, '

   106         'X meta M, X symetric S, X inlined I', build_descr=False):

   107         try:

   108             # bw compat: fulltext_container added in 2.47

   109             ft_container = session.execute('Any FTC WHERE X eid %(x)s, X fulltext_container FTC',

   110                                            {'x': eid}).rows[0][0]

   111         except:

   112             ft_container = None

   113             session.rollback(False)

   114         rtype = ybo.RelationType(name=rtype, description=desc, meta=bool(meta),

   115                                  symetric=bool(sym), inlined=bool(il),

   116                                  fulltext_container=ft_container, eid=eid)

   117         rschema = schema.add_relation_type(rtype)

   118         index[eid] = rschema

   119         set_perms(rschema, permsdict.get(eid, {}))        

   120     cstrsdict = deserialize_rdef_constraints(session)

   121     for values in session.execute(

   122         'Any X,SE,RT,OE,CARD,ORD,DESC,IDX,FTIDX,I18N,DFLT WHERE X is EFRDef,'

   123         'X relation_type RT, X cardinality CARD, X ordernum ORD, X indexed IDX,'

   124         'X description DESC, X internationalizable I18N, X defaultval DFLT,'

   125         'X fulltextindexed FTIDX, X from_entity SE, X to_entity OE',

   126         build_descr=False):

   127         rdefeid, seid, reid, teid, card, ord, desc, idx, ftidx, i18n, default = values

   128         constraints = cstrsdict.get(rdefeid, ())

   129         frometype = index[seid].type

   130         rtype = index[reid].type

   131         toetype = index[teid].type

   132         rdef = ybo.RelationDefinition(frometype, rtype, toetype, cardinality=card,

   133                                   order=ord, description=desc, 

   134                                   constraints=constraints,

   135                                   indexed=idx, fulltextindexed=ftidx,

   136                                   internationalizable=i18n,

   137                                   default=default, eid=rdefeid)

   138         schema.add_relation_def(rdef)

   139     for values in session.execute(

   140         'Any X,SE,RT,OE,CARD,ORD,DESC,C WHERE X is ENFRDef, X relation_type RT,'

   141         'X cardinality CARD, X ordernum ORD, X description DESC, '

   142         'X from_entity SE, X to_entity OE, X composite C', build_descr=False):

   143         rdefeid, seid, reid, teid, card, ord, desc, c = values

   144         frometype = index[seid].type

   145         rtype = index[reid].type

   146         toetype = index[teid].type

   147         constraints = cstrsdict.get(rdefeid, ())

   148         rdef = ybo.RelationDefinition(frometype, rtype, toetype, cardinality=card,

   149                                   order=ord, description=desc, 

   150                                   composite=c, constraints=constraints,

   151                                   eid=rdefeid)

   152         schema.add_relation_def(rdef)

   153     schema.infer_specialization_rules()

   154     session.commit()

   155     schema.reading_from_database = False

   156 

   157 

   158 def deserialize_ertype_permissions(session):

   159     """return sect action:groups associations for the given

   160     entity or relation schema with its eid, according to schema's

   161     permissions stored in the database as [read|add|delete|update]_permission

   162     relations between EEType/ERType and EGroup entities

   163     """

   164     res = {}

   165     for action in ('read', 'add', 'update', 'delete'):

   166         rql = 'Any E,N WHERE G is EGroup, G name N, E %s_permission G' % action

   167         for eid, gname in session.execute(rql, build_descr=False):

   168             res.setdefault(eid, {}).setdefault(action, []).append(gname)

   169         rql = ('Any E,X,EXPR,V WHERE X is RQLExpression, X expression EXPR, '

   170                'E %s_permission X, X mainvars V' % action)

   171         for eid, expreid, expr, mainvars in session.execute(rql, build_descr=False):

   172             # we don't know yet if it's a rql expr for an entity or a relation,

   173             # so append a tuple to differentiate from groups and so we'll be

   174             # able to instantiate it later

   175             res.setdefault(eid, {}).setdefault(action, []).append( (expr, mainvars, expreid) )

   176     return res

   177 

   178 def set_perms(erschema, permsdict):

   179     """set permissions on the given erschema according to the permission

   180     definition dictionary as built by deserialize_ertype_permissions for a

   181     given erschema's eid

   182     """

   183     for action in erschema.ACTIONS:

   184         actperms = []

   185         for something in permsdict.get(action, ()):

   186             if isinstance(something, tuple):

   187                 actperms.append(erschema.rql_expression(*something))

   188             else: # group name

   189                 actperms.append(something)

   190         erschema.set_permissions(action, actperms)            

   191 

   192 

   193 def deserialize_rdef_constraints(session):

   194     """return the list of relation definition's constraints as instances"""

   195     res = {}

   196     for rdefeid, ceid, ct, val in session.execute(

   197         'Any E, X,TN,V WHERE E constrained_by X, X is EConstraint, '

   198         'X cstrtype T, T name TN, X value V', build_descr=False):

   199         cstr = CONSTRAINTS[ct].deserialize(val)

   200         cstr.eid = ceid

   201         res.setdefault(rdefeid, []).append(cstr)

   202     return res

   203         

   204         

   205 # schema / perms serialization ################################################

   206 

   207 def serialize_schema(cursor, schema, verbose=False):

   208     """synchronize schema and permissions in the database according to

   209     current schema

   210     """

   211     print 'serializing the schema, this may take some time'

   212     eschemas = schema.entities()

   213     aller = eschemas + schema.relations()

   214     if not verbose:

   215         pb_size = len(aller) + len(CONSTRAINTS) + len([x for x in eschemas if x.specializes()])

   216         pb = ProgressBar(pb_size)

   217     for cstrtype in CONSTRAINTS:

   218         rql = 'INSERT EConstraintType X: X name "%s"' % cstrtype

   219         if verbose:

   220             print rql

   221         cursor.execute(rql)

   222         if not verbose:

   223             pb.update()

   224     groupmap = group_mapping(cursor, interactive=False)

   225     for ertype in aller:

   226         # skip eid and has_text relations

   227         if ertype in ('eid', 'identity', 'has_text',):

   228             pb.update()

   229             continue

   230         for rql, kwargs in erschema2rql(schema[ertype]):

   231             if verbose:

   232                 print rql % kwargs

   233             cursor.execute(rql, kwargs)

   234         for rql, kwargs in erperms2rql(schema[ertype], groupmap):

   235             if verbose:

   236                 print rql

   237             cursor.execute(rql, kwargs)

   238         if not verbose:

   239             pb.update()

   240     for rql, kwargs in specialize2rql(schema):

   241         if verbose:

   242             print rql % kwargs

   243         cursor.execute(rql, kwargs)

   244         if not verbose:

   245             pb.update()

   246     print

   247 

   248 

   249 def _ervalues(erschema):

   250     try:

   251         type_ = unicode(erschema.type)

   252     except UnicodeDecodeError, e:

   253         raise Exception("can't decode %s [was %s]" % (erschema.type, e))

   254     try:

   255         desc = unicode(erschema.description) or u''

   256     except UnicodeDecodeError, e:

   257         raise Exception("can't decode %s [was %s]" % (erschema.description, e))

   258     return {

   259         'name': type_,

   260         'meta': erschema.meta,

   261         'final': erschema.is_final(),

   262         'description': desc,

   263         }

   264 

   265 def eschema_relations_values(eschema):

   266     values = _ervalues(eschema)

   267     relations = ['X %s %%(%s)s' % (attr, attr) for attr in sorted(values)]

   268     return relations, values

   269 

   270 # XXX 2.47 migration

   271 HAS_FULLTEXT_CONTAINER = True

   272 

   273 def rschema_relations_values(rschema):

   274     values = _ervalues(rschema)

   275     values['final'] = rschema.is_final()

   276     values['symetric'] = rschema.symetric

   277     values['inlined'] = rschema.inlined

   278     if HAS_FULLTEXT_CONTAINER:

   279         if isinstance(rschema.fulltext_container, str):

   280             values['fulltext_container'] = unicode(rschema.fulltext_container)

   281         else:

   282             values['fulltext_container'] = rschema.fulltext_container

   283     relations = ['X %s %%(%s)s' % (attr, attr) for attr in sorted(values)]

   284     return relations, values

   285 

   286 def _rdef_values(rschema, objtype, props):

   287     amap = {'order': 'ordernum'}

   288     values = {}

   289     for prop, default in rschema.rproperty_defs(objtype).iteritems():

   290         if prop in ('eid', 'constraints', 'uid', 'infered'):

   291             continue

   292         value = props.get(prop, default)

   293         if prop in ('indexed', 'fulltextindexed', 'internationalizable'):

   294             value = bool(value)

   295         elif prop == 'ordernum':

   296             value = int(value)

   297         elif isinstance(value, str):

   298             value = unicode(value)

   299         values[amap.get(prop, prop)] = value

   300     return values

   301     

   302 def nfrdef_relations_values(rschema, objtype, props):

   303     values = _rdef_values(rschema, objtype, props)

   304     relations = ['X %s %%(%s)s' % (attr, attr) for attr in sorted(values)]

   305     return relations, values

   306     

   307 def frdef_relations_values(rschema, objtype, props):

   308     values = _rdef_values(rschema, objtype, props)

   309     default = values['default']

   310     del values['default']

   311     if default is not None:

   312         if default is False:

   313             default = u''

   314         elif not isinstance(default, unicode):

   315             default = unicode(default)

   316     values['defaultval'] = default

   317     relations = ['X %s %%(%s)s' % (attr, attr) for attr in sorted(values)]

   318     return relations, values

   319 

   320     

   321 def __rdef2rql(genmap, rschema, subjtype=None, objtype=None, props=None):

   322     if subjtype is None:

   323         assert objtype is None

   324         assert props is None

   325         targets = rschema.iter_rdefs()

   326     else:

   327         assert not objtype is None

   328         targets = [(subjtype, objtype)]

   329     for subjtype, objtype in targets:

   330         if props is None:

   331             _props = rschema.rproperties(subjtype, objtype)

   332         else:

   333             _props = props

   334         # don't serialize infered relations

   335         if _props.get('infered'):

   336             continue

   337         gen = genmap[rschema.is_final()]

   338         for rql, values in gen(rschema, subjtype, objtype, _props):

   339             yield rql, values

   340 

   341 

   342 def schema2rql(schema, skip=None, allow=None):

   343     """return a list of rql insert statements to enter the schema in the

   344     database as ERType and EEType entities

   345     """

   346     assert not (skip is not None and allow is not None), \

   347            'can\'t use both skip and allow'

   348     all = schema.entities() + schema.relations()

   349     if skip is not None:

   350         return chain(*[erschema2rql(schema[t]) for t in all if not t in skip])

   351     elif allow is not None:

   352         return chain(*[erschema2rql(schema[t]) for t in all if t in allow])

   353     return chain(*[erschema2rql(schema[t]) for t in all])

   354         

   355 def erschema2rql(erschema):

   356     if isinstance(erschema, schemamod.EntitySchema):

   357         return eschema2rql(erschema)

   358     return rschema2rql(erschema)

   359 

   360 def eschema2rql(eschema):

   361     """return a list of rql insert statements to enter an entity schema

   362     in the database as an EEType entity

   363     """

   364     relations, values = eschema_relations_values(eschema)

   365     # NOTE: 'specializes' relation can't be inserted here since there's no

   366     # way to make sure the parent type is inserted before the child type

   367     yield 'INSERT EEType X: %s' % ','.join(relations) , values

   368 

   369 def specialize2rql(schema):

   370     for eschema in schema.entities():

   371         for rql, kwargs in eschemaspecialize2rql(eschema):

   372             yield rql, kwargs

   373 

   374 def eschemaspecialize2rql(eschema):

   375     specialized_type = eschema.specializes()

   376     if specialized_type:

   377         values = {'x': eschema.type, 'et': specialized_type.type}

   378         yield 'SET X specializes ET WHERE X name %(x)s, ET name %(et)s', values

   379 

   380 def rschema2rql(rschema, addrdef=True):

   381     """return a list of rql insert statements to enter a relation schema

   382     in the database as an ERType entity

   383     """

   384     if rschema.type == 'has_text':

   385         return

   386     relations, values = rschema_relations_values(rschema)

   387     yield 'INSERT ERType X: %s' % ','.join(relations), values

   388     if addrdef:

   389         for rql, values in rdef2rql(rschema):

   390             yield rql, values

   391             

   392 def rdef2rql(rschema, subjtype=None, objtype=None, props=None):

   393     genmap = {True: frdef2rql, False: nfrdef2rql}

   394     return __rdef2rql(genmap, rschema, subjtype, objtype, props)

   395 

   396 

   397 _LOCATE_RDEF_RQL0 = 'X relation_type ER,X from_entity SE,X to_entity OE'

   398 _LOCATE_RDEF_RQL1 = 'SE name %(se)s,ER name %(rt)s,OE name %(oe)s'

   399 

   400 def frdef2rql(rschema, subjtype, objtype, props):

   401     relations, values = frdef_relations_values(rschema, objtype, props)

   402     relations.append(_LOCATE_RDEF_RQL0)

   403     values.update({'se': str(subjtype), 'rt': str(rschema), 'oe': str(objtype)})

   404     yield 'INSERT EFRDef X: %s WHERE %s' % (','.join(relations), _LOCATE_RDEF_RQL1), values

   405     for rql, values in rdefrelations2rql(rschema, subjtype, objtype, props):

   406         yield rql + ', EDEF is EFRDef', values

   407             

   408 def nfrdef2rql(rschema, subjtype, objtype, props):

   409     relations, values = nfrdef_relations_values(rschema, objtype, props)

   410     relations.append(_LOCATE_RDEF_RQL0)

   411     values.update({'se': str(subjtype), 'rt': str(rschema), 'oe': str(objtype)})

   412     yield 'INSERT ENFRDef X: %s WHERE %s' % (','.join(relations), _LOCATE_RDEF_RQL1), values

   413     for rql, values in rdefrelations2rql(rschema, subjtype, objtype, props):

   414         yield rql + ', EDEF is ENFRDef', values

   415                 

   416 def rdefrelations2rql(rschema, subjtype, objtype, props):

   417     iterators = []

   418     for constraint in props['constraints']:

   419         iterators.append(constraint2rql(rschema, subjtype, objtype, constraint))

   420     return chain(*iterators)

   421 

   422 def constraint2rql(rschema, subjtype, objtype, constraint):

   423     values = {'ctname': unicode(constraint.type()),

   424               'value': unicode(constraint.serialize()),

   425               'rt': str(rschema), 'se': str(subjtype), 'oe': str(objtype)}

   426     yield 'INSERT EConstraint X: X value %(value)s, X cstrtype CT, EDEF constrained_by X WHERE \

   427 CT name %(ctname)s, EDEF relation_type ER, EDEF from_entity SE, EDEF to_entity OE, \

   428 ER name %(rt)s, SE name %(se)s, OE name %(oe)s', values

   429 

   430 def perms2rql(schema, groupmapping):

   431     """return rql insert statements to enter the schema's permissions in

   432     the database as [read|add|delete|update]_permission relations between

   433     EEType/ERType and EGroup entities

   434 

   435     groupmapping is a dictionnary mapping standard group names to

   436     eids

   437     """

   438     for etype in sorted(schema.entities()):

   439         yield erperms2rql(schema[etype], groupmapping)

   440     for rtype in sorted(schema.relations()):

   441         yield erperms2rql(schema[rtype], groupmapping)

   442 

   443 def erperms2rql(erschema, groupmapping):

   444     """return rql insert statements to enter the entity or relation

   445     schema's permissions in the database as

   446     [read|add|delete|update]_permission relations between EEType/ERType

   447     and EGroup entities

   448     """

   449     etype = isinstance(erschema, schemamod.EntitySchema) and 'EEType' or 'ERType'

   450     for action in erschema.ACTIONS:

   451         for group in sorted(erschema.get_groups(action)):

   452             try:

   453                 yield ('SET X %s_permission Y WHERE X is %s, X name "%s", Y eid %s'

   454                        % (action, etype, erschema, groupmapping[group]), None)

   455             except KeyError:

   456                 continue

   457         for rqlexpr in sorted(erschema.get_rqlexprs(action)):

   458             yield ('INSERT RQLExpression E: E expression %%(e)s, E exprtype %%(t)s, '

   459                    'E mainvars %%(v)s, X %s_permission E '

   460                    'WHERE X is %s, X name "%s"' % (action, etype, erschema),

   461                    {'e': unicode(rqlexpr.expression), 'v': unicode(rqlexpr.mainvars),

   462                     't': unicode(rqlexpr.__class__.__name__)})

   463 

   464 

   465 def updateeschema2rql(eschema):

   466     relations, values = eschema_relations_values(eschema)

   467     values['et'] = eschema.type

   468     yield 'SET %s WHERE X is EEType, X name %%(et)s' % ','.join(relations), values

   469 

   470 def updaterschema2rql(rschema):

   471     relations, values = rschema_relations_values(rschema)

   472     values['rt'] = rschema.type

   473     yield 'SET %s WHERE X is ERType, X name %%(rt)s' % ','.join(relations), values

   474             

   475 def updaterdef2rql(rschema, subjtype=None, objtype=None, props=None):

   476     genmap = {True: updatefrdef2rql, False: updatenfrdef2rql}

   477     return __rdef2rql(genmap, rschema, subjtype, objtype, props)

   478 

   479 def updatefrdef2rql(rschema, subjtype, objtype, props):

   480     relations, values = frdef_relations_values(rschema, objtype, props)

   481     values.update({'se': subjtype, 'rt': str(rschema), 'oe': objtype})

   482     yield 'SET %s WHERE %s, %s, X is EFRDef' % (','.join(relations),

   483                                                  _LOCATE_RDEF_RQL0,

   484                                                  _LOCATE_RDEF_RQL1), values

   485             

   486 def updatenfrdef2rql(rschema, subjtype, objtype, props):

   487     relations, values = nfrdef_relations_values(rschema, objtype, props)

   488     values.update({'se': subjtype, 'rt': str(rschema), 'oe': objtype})

   489     yield 'SET %s WHERE %s, %s, X is ENFRDef' % (','.join(relations),

   490                                                  _LOCATE_RDEF_RQL0,

   491                                                  _LOCATE_RDEF_RQL1), values