1 # copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.

     2 # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr

     3 #

     4 # This file is part of CubicWeb.

     5 #

     6 # CubicWeb is free software: you can redistribute it and/or modify it under the

     7 # terms of the GNU Lesser General Public License as published by the Free

     8 # Software Foundation, either version 2.1 of the License, or (at your option)

     9 # any later version.

    10 #

    11 # CubicWeb is distributed in the hope that it will be useful, but WITHOUT

    12 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS

    13 # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more

    14 # details.

    15 #

    16 # You should have received a copy of the GNU Lesser General Public License along

    17 # with CubicWeb.  If not, see <http://www.gnu.org/licenses/>.

    18 """allways executed before all others in server migration

    19 

    20 it should only include low level schema changes

    21 """

    22 from __future__ import print_function

    23 

    24 from six import text_type

    25 

    26 from cubicweb import ConfigurationError

    27 from cubicweb.server.session import hooks_control

    28 from cubicweb.server import schemaserial as ss

    29 

    30 applcubicwebversion, cubicwebversion = versions_map['cubicweb']

    31 

    32 def _add_relation_definition_no_perms(subjtype, rtype, objtype):

    33     rschema = fsschema.rschema(rtype)

    34     rdef = rschema.rdefs[(subjtype, objtype)]

    35     rdef.rtype = schema.rschema(rtype)

    36     rdef.subject = schema.eschema(subjtype)

    37     rdef.object = schema.eschema(objtype)

    38     ss.execschemarql(rql, rdef, ss.rdef2rql(rdef, CSTRMAP, groupmap=None))

    39     commit(ask_confirm=False)

    40 

    41 def replace_eid_sequence_with_eid_numrange(session):

    42     dbh = session.repo.system_source.dbhelper

    43     cursor = session.cnxset.cu

    44     try:

    45         cursor.execute(dbh.sql_sequence_current_state('entities_id_seq'))

    46         lasteid = cursor.fetchone()[0]

    47     except: # programming error, already migrated

    48         return

    49 

    50     cursor.execute(dbh.sql_drop_sequence('entities_id_seq'))

    51     cursor.execute(dbh.sql_create_numrange('entities_id_seq'))

    52     cursor.execute(dbh.sql_restart_numrange('entities_id_seq', initial_value=lasteid))

    53     session.commit()

    54 

    55 if applcubicwebversion <= (3, 13, 0) and cubicwebversion >= (3, 13, 1):

    56     sql('ALTER TABLE entities ADD asource VARCHAR(64)')

    57     sql('UPDATE entities SET asource=cw_name  '

    58         'FROM cw_CWSource, cw_source_relation '

    59         'WHERE entities.eid=cw_source_relation.eid_from AND cw_source_relation.eid_to=cw_CWSource.cw_eid')

    60     commit()

    61 

    62 if applcubicwebversion <= (3, 14, 4) and cubicwebversion >= (3, 14, 4):

    63     from cubicweb.server import schema2sql as y2sql

    64     dbhelper = repo.system_source.dbhelper

    65     rdefdef = schema['CWSource'].rdef('name')

    66     attrtype = y2sql.type_from_constraints(dbhelper, rdefdef.object, rdefdef.constraints).split()[0]

    67     cursor = session.cnxset.cu

    68     sql('UPDATE entities SET asource = source WHERE asource is NULL')

    69     dbhelper.change_col_type(cursor, 'entities', 'asource', attrtype, False)

    70     dbhelper.change_col_type(cursor, 'entities', 'source', attrtype, False)

    71 

    72     # we now have a functional asource column, start using the normal eid_type_source method

    73     if repo.system_source.eid_type_source == repo.system_source.eid_type_source_pre_131:

    74         del repo.system_source.eid_type_source

    75 

    76 if applcubicwebversion < (3, 19, 0) and cubicwebversion >= (3, 19, 0):

    77     try: 

    78         # need explicit drop of the indexes on some database systems (sqlserver)

    79         sql(repo.system_source.dbhelper.sql_drop_index('entities', 'mtime'))

    80         sql('ALTER TABLE "entities" DROP COLUMN "mtime"')

    81         sql('ALTER TABLE "entities" DROP COLUMN "source"')

    82     except: # programming error, already migrated

    83         print("Failed to drop mtime or source database columns")

    84         print("'entities' table of the database has probably been already updated")

    85 

    86     commit()

    87 

    88     replace_eid_sequence_with_eid_numrange(session)

    89 

    90 if applcubicwebversion < (3, 20, 0) and cubicwebversion >= (3, 20, 0):

    91     ss._IGNORED_PROPS.append('formula')

    92     add_attribute('CWAttribute', 'formula', commit=False)

    93     ss._IGNORED_PROPS.remove('formula')

    94     commit()

    95     add_entity_type('CWComputedRType')

    96     commit()

    97 

    98 if schema['TZDatetime'].eid is None:

    99     add_entity_type('TZDatetime', auto=False)

   100 if schema['TZTime'].eid is None:

   101     add_entity_type('TZTime', auto=False)

   102 

   103 if applcubicwebversion < (3, 18, 0) and cubicwebversion >= (3, 18, 0):

   104     driver = config.system_source_config['db-driver']

   105     if not (driver == 'postgres' or driver.startswith('sqlserver')):

   106         import sys

   107         print('This migration is not supported for backends other than sqlserver or postgres (yet).', file=sys.stderr)

   108         sys.exit(1)

   109 

   110     add_relation_definition('CWAttribute', 'add_permission', 'CWGroup')

   111     add_relation_definition('CWAttribute', 'add_permission', 'RQLExpression')

   112 

   113     # a bad defaultval in 3.13.8 schema was fixed in 3.13.9, but the migration was missed

   114     rql('SET ATTR defaultval NULL WHERE ATTR from_entity E, E name "CWSource", ATTR relation_type T, T name "in_synchronization"')

   115 

   116     # the migration gets confused when we change rdefs out from under it.  So

   117     # explicitly remove this size constraint so it doesn't stick around and break

   118     # things later.

   119     rdefeid = schema['defaultval'].rdefs.values()[0].eid

   120     rql('DELETE CWConstraint C WHERE C cstrtype T, T name "SizeConstraint", R constrained_by C, R eid %(eid)s', {'eid': rdefeid})

   121 

   122     sync_schema_props_perms('defaultval')

   123 

   124     def convert_defaultval(cwattr, default):

   125         from decimal import Decimal

   126         import yams

   127         from cubicweb import Binary

   128         if default is None:

   129             return

   130         if isinstance(default, Binary):

   131             # partially migrated instance, try to be idempotent

   132             return default

   133         atype = cwattr.to_entity[0].name

   134         if atype == 'Boolean':

   135             # boolean attributes with default=False were stored as ''

   136             assert default in ('True', 'False', ''), repr(default)

   137             default = default == 'True'

   138         elif atype in ('Int', 'BigInt'):

   139             default = int(default)

   140         elif atype == 'Float':

   141             default = float(default)

   142         elif atype == 'Decimal':

   143             default = Decimal(default)

   144         elif atype in ('Date', 'Datetime', 'TZDatetime', 'Time'):

   145             try:

   146                 # handle NOW and TODAY, keep them stored as strings

   147                 yams.KEYWORD_MAP[atype][default.upper()]

   148                 default = default.upper()

   149             except KeyError:

   150                 # otherwise get an actual date or datetime

   151                 default = yams.DATE_FACTORY_MAP[atype](default)

   152         else:

   153             assert atype == 'String', atype

   154             default = text_type(default)

   155         return Binary.zpickle(default)

   156 

   157     dbh = repo.system_source.dbhelper

   158 

   159 

   160     sql('ALTER TABLE cw_cwattribute ADD new_defaultval %s' % dbh.TYPE_MAPPING['Bytes'])

   161 

   162     for cwattr in rql('CWAttribute X').entities():

   163         olddefault = cwattr.defaultval

   164         if olddefault is not None:

   165             req = "UPDATE cw_cwattribute SET new_defaultval = %(val)s WHERE cw_eid = %(eid)s"

   166             args = {'val': dbh.binary_value(convert_defaultval(cwattr, olddefault).getvalue()), 'eid': cwattr.eid}

   167             sql(req, args, ask_confirm=False)

   168 

   169     sql('ALTER TABLE cw_cwattribute DROP COLUMN cw_defaultval')

   170     if driver == 'postgres':

   171         sql('ALTER TABLE cw_cwattribute RENAME COLUMN new_defaultval TO cw_defaultval')

   172     else: # sqlserver

   173         sql("sp_rename 'cw_cwattribute.new_defaultval', 'cw_defaultval', 'COLUMN'")

   174 

   175 

   176     # Set object type to "Bytes" for CWAttribute's "defaultval" attribute

   177     rql('SET X to_entity B WHERE X is CWAttribute, X from_entity Y, Y name "CWAttribute", '

   178         'X relation_type Z, Z name "defaultval", B name "Bytes", NOT X to_entity B')

   179 

   180     oldrdef = schema['CWAttribute'].rdef('defaultval')

   181     import yams.buildobjs as ybo

   182     newrdef = ybo.RelationDefinition('CWAttribute', 'defaultval', 'Bytes')

   183     newrdef.eid = oldrdef.eid

   184     schema.add_relation_def(newrdef)

   185     schema.del_relation_def('CWAttribute', 'defaultval', 'String')

   186 

   187     commit()

   188 

   189     sync_schema_props_perms('defaultval')

   190 

   191     for rschema in schema.relations():

   192         if rschema.symmetric:

   193             subjects = set(repr(e.type) for e in rschema.subjects())

   194             objects = set(repr(e.type) for e in rschema.objects())

   195             assert subjects == objects

   196             martians = set(str(eid) for eid, in sql('SELECT eid_to FROM %s_relation, entities WHERE eid_to = eid AND type NOT IN (%s)' %

   197                                                (rschema.type, ','.join(subjects))))

   198             martians |= set(str(eid) for eid, in sql('SELECT eid_from FROM %s_relation, entities WHERE eid_from = eid AND type NOT IN (%s)' %

   199                                                 (rschema.type, ','.join(subjects))))

   200             if martians:

   201                 martians = ','.join(martians)

   202                 print('deleting broken relations %s for eids %s' % (rschema.type, martians))

   203                 sql('DELETE FROM %s_relation WHERE eid_from IN (%s) OR eid_to IN (%s)' % (rschema.type, martians, martians))

   204             with session.deny_all_hooks_but():

   205                 rql('SET X %(r)s Y WHERE Y %(r)s X, NOT X %(r)s Y' % {'r': rschema.type})

   206             commit()

   207 

   208 

   209     # multi columns unique constraints regeneration

   210     from cubicweb.server import schemaserial

   211 

   212     # syncschema hooks would try to remove indices but

   213     # 1) we already do that below

   214     # 2) the hook expects the CWUniqueTogetherConstraint.name attribute that hasn't

   215     #    yet been added

   216     with session.allow_all_hooks_but('syncschema'):

   217         rql('DELETE CWUniqueTogetherConstraint C')

   218     commit()

   219     add_attribute('CWUniqueTogetherConstraint', 'name')

   220 

   221     # low-level wipe code for postgres & sqlserver, plain sql ...

   222     if driver == 'postgres':

   223         for indexname, in sql('select indexname from pg_indexes'):

   224             if indexname.startswith('unique_'):

   225                 print('dropping index', indexname)

   226                 sql('DROP INDEX %s' % indexname)

   227         commit()

   228     elif driver.startswith('sqlserver'):

   229         for viewname, in sql('select name from sys.views'):

   230             if viewname.startswith('utv_'):

   231                 print('dropping view (index should be cascade-deleted)', viewname)

   232                 sql('DROP VIEW %s' % viewname)

   233         commit()

   234 

   235     # recreate the constraints, hook will lead to low-level recreation

   236     for eschema in sorted(schema.entities()):

   237         if eschema._unique_together:

   238             print('recreate unique indexes for', eschema)

   239             rql_args = schemaserial.uniquetogether2rqls(eschema)

   240             for rql, args in rql_args:

   241                 args['x'] = eschema.eid

   242                 session.execute(rql, args)

   243     commit()

   244 

   245     # all attributes perms have to be refreshed ...

   246     for rschema in sorted(schema.relations()):

   247         if rschema.final:

   248             if rschema.type in fsschema:

   249                 print('sync perms for', rschema.type)

   250                 sync_schema_props_perms(rschema.type, syncprops=False, ask_confirm=False, commit=False)

   251             else:

   252                 print('WARNING: attribute %s missing from fs schema' % rschema.type)

   253     commit()

   254 

   255 if applcubicwebversion < (3, 17, 0) and cubicwebversion >= (3, 17, 0):

   256     try:

   257         add_cube('sioc', update_database=False)

   258     except ConfigurationError:

   259         if not confirm('In cubicweb 3.17 sioc views have been moved to the sioc '

   260                        'cube, which is not installed.  Continue anyway?'):

   261             raise

   262     try:

   263         add_cube('embed', update_database=False)

   264     except ConfigurationError:

   265         if not confirm('In cubicweb 3.17 embedding views have been moved to the embed '

   266                        'cube, which is not installed.  Continue anyway?'):

   267             raise

   268     try:

   269         add_cube('geocoding', update_database=False)

   270     except ConfigurationError:

   271         if not confirm('In cubicweb 3.17 geocoding views have been moved to the geocoding '

   272                        'cube, which is not installed.  Continue anyway?'):

   273             raise

   274 

   275 

   276 if applcubicwebversion <= (3, 14, 0) and cubicwebversion >= (3, 14, 0):

   277     if 'require_permission' in schema and not 'localperms'in repo.config.cubes():

   278         from cubicweb import ExecutionError

   279         try:

   280             add_cube('localperms', update_database=False)

   281         except ConfigurationError:

   282             raise ExecutionError('In cubicweb 3.14, CWPermission and related stuff '

   283                                  'has been moved to cube localperms. Install it first.')

   284 

   285 

   286 if applcubicwebversion == (3, 6, 0) and cubicwebversion >= (3, 6, 0):

   287     CSTRMAP = dict(rql('Any T, X WHERE X is CWConstraintType, X name T',

   288                        ask_confirm=False))

   289     _add_relation_definition_no_perms('CWAttribute', 'update_permission', 'CWGroup')

   290     _add_relation_definition_no_perms('CWAttribute', 'update_permission', 'RQLExpression')

   291     rql('SET X update_permission Y WHERE X is CWAttribute, X add_permission Y')

   292     drop_relation_definition('CWAttribute', 'delete_permission', 'CWGroup')

   293     drop_relation_definition('CWAttribute', 'delete_permission', 'RQLExpression')

   294 

   295 elif applcubicwebversion < (3, 6, 0) and cubicwebversion >= (3, 6, 0):

   296     CSTRMAP = dict(rql('Any T, X WHERE X is CWConstraintType, X name T',

   297                        ask_confirm=False))

   298     session.set_cnxset()

   299     permsdict = ss.deserialize_ertype_permissions(session)

   300 

   301     with hooks_control(session, session.HOOKS_ALLOW_ALL, 'integrity'):

   302         for rschema in repo.schema.relations():

   303             rpermsdict = permsdict.get(rschema.eid, {})

   304             for rdef in rschema.rdefs.values():

   305                 for action in rdef.ACTIONS:

   306                     actperms = []

   307                     for something in rpermsdict.get(action == 'update' and 'add' or action, ()):

   308                         if isinstance(something, tuple):

   309                             actperms.append(rdef.rql_expression(*something))

   310                         else: # group name

   311                             actperms.append(something)

   312                     rdef.set_action_permissions(action, actperms)

   313         for action in ('read', 'add', 'delete'):

   314             _add_relation_definition_no_perms('CWRelation', '%s_permission' % action, 'CWGroup')

   315             _add_relation_definition_no_perms('CWRelation', '%s_permission' % action, 'RQLExpression')

   316         for action in ('read', 'update'):

   317             _add_relation_definition_no_perms('CWAttribute', '%s_permission' % action, 'CWGroup')

   318             _add_relation_definition_no_perms('CWAttribute', '%s_permission' % action, 'RQLExpression')

   319         for action in ('read', 'add', 'delete'):

   320             rql('SET X %s_permission Y WHERE X is CWRelation, '

   321                 'RT %s_permission Y, X relation_type RT, Y is CWGroup' % (action, action))

   322             rql('INSERT RQLExpression Y: Y exprtype YET, Y mainvars YMV, Y expression YEX, '

   323                 'X %s_permission Y WHERE X is CWRelation, '

   324                 'X relation_type RT, RT %s_permission Y2, Y2 exprtype YET, '

   325                 'Y2 mainvars YMV, Y2 expression YEX' % (action, action))

   326         rql('SET X read_permission Y WHERE X is CWAttribute, '

   327             'RT read_permission Y, X relation_type RT, Y is CWGroup')

   328         rql('INSERT RQLExpression Y: Y exprtype YET, Y mainvars YMV, Y expression YEX, '

   329             'X read_permission Y WHERE X is CWAttribute, '

   330             'X relation_type RT, RT read_permission Y2, Y2 exprtype YET, '

   331             'Y2 mainvars YMV, Y2 expression YEX')

   332         rql('SET X update_permission Y WHERE X is CWAttribute, '

   333             'RT add_permission Y, X relation_type RT, Y is CWGroup')

   334         rql('INSERT RQLExpression Y: Y exprtype YET, Y mainvars YMV, Y expression YEX, '

   335             'X update_permission Y WHERE X is CWAttribute, '

   336             'X relation_type RT, RT add_permission Y2, Y2 exprtype YET, '

   337             'Y2 mainvars YMV, Y2 expression YEX')

   338         for action in ('read', 'add', 'delete'):

   339             drop_relation_definition('CWRType', '%s_permission' % action, 'CWGroup', commit=False)

   340             drop_relation_definition('CWRType', '%s_permission' % action, 'RQLExpression')

   341     sync_schema_props_perms('read_permission', syncperms=False) # fix read_permission cardinality

   342 

   343 if applcubicwebversion < (3, 9, 6) and cubicwebversion >= (3, 9, 6) and not 'CWUniqueTogetherConstraint' in schema:

   344     add_entity_type('CWUniqueTogetherConstraint')

   345 

   346 if not ('CWUniqueTogetherConstraint', 'CWRType') in schema['relations'].rdefs:

   347     add_relation_definition('CWUniqueTogetherConstraint', 'relations', 'CWRType')

   348     rql('SET C relations RT WHERE C relations RDEF, RDEF relation_type RT')

   349     commit()

   350     drop_relation_definition('CWUniqueTogetherConstraint', 'relations', 'CWAttribute')

   351     drop_relation_definition('CWUniqueTogetherConstraint', 'relations', 'CWRelation')

   352 

   353 

   354 if applcubicwebversion < (3, 4, 0) and cubicwebversion >= (3, 4, 0):

   355 

   356     with hooks_control(session, session.HOOKS_ALLOW_ALL, 'integrity'):

   357         session.set_shared_data('do-not-insert-cwuri', True)

   358         add_relation_type('cwuri')

   359         base_url = session.base_url()

   360         for eid, in rql('Any X', ask_confirm=False):

   361             type, source, extid = session.describe(eid)

   362             if source == 'system':

   363                 rql('SET X cwuri %(u)s WHERE X eid %(x)s',

   364                     {'x': eid, 'u': u'%s%s' % (base_url, eid)})

   365         isession.commit()

   366         session.set_shared_data('do-not-insert-cwuri', False)

   367 

   368 if applcubicwebversion < (3, 5, 0) and cubicwebversion >= (3, 5, 0):

   369     # check that migration is not doomed

   370     rset = rql('Any X,Y WHERE X transition_of E, Y transition_of E, '

   371                'X name N, Y name N, NOT X identity Y',

   372                ask_confirm=False)

   373     if rset:

   374         from logilab.common.shellutils import ASK

   375         if not ASK.confirm('Migration will fail because of transitions with the same name. '

   376                            'Continue anyway ?'):

   377             import sys

   378             sys.exit(1)

   379     # proceed with migration

   380     add_entity_type('Workflow')

   381     add_entity_type('BaseTransition')

   382     add_entity_type('WorkflowTransition')

   383     add_entity_type('SubWorkflowExitPoint')

   384     # drop explicit 'State allowed_transition Transition' since it should be

   385     # infered due to yams inheritance.  However we've to disable the schema

   386     # sync hook first to avoid to destroy existing data...

   387     try:

   388         from cubicweb.hooks import syncschema

   389         repo.vreg.unregister(syncschema.AfterDelRelationTypeHook)

   390         try:

   391             drop_relation_definition('State', 'allowed_transition', 'Transition')

   392         finally:

   393             repo.vreg.register(syncschema.AfterDelRelationTypeHook)

   394     except ImportError: # syncschema is in CW >= 3.6 only

   395         from cubicweb.server.schemahooks import after_del_relation_type

   396         repo.hm.unregister_hook(after_del_relation_type,

   397                                 'after_delete_relation', 'relation_type')

   398         try:

   399             drop_relation_definition('State', 'allowed_transition', 'Transition')

   400         finally:

   401             repo.hm.register_hook(after_del_relation_type,

   402                                   'after_delete_relation', 'relation_type')

   403     schema.rebuild_infered_relations() # need to be explicitly called once everything is in place

   404 

   405     for et in rql('DISTINCT Any ET,ETN WHERE S state_of ET, ET name ETN',

   406                   ask_confirm=False).entities():

   407         wf = add_workflow(u'default %s workflow' % et.name, et.name,

   408                           ask_confirm=False)

   409         rql('SET S state_of WF WHERE S state_of ET, ET eid %(et)s, WF eid %(wf)s',

   410             {'et': et.eid, 'wf': wf.eid}, 'et', ask_confirm=False)

   411         rql('SET T transition_of WF WHERE T transition_of ET, ET eid %(et)s, WF eid %(wf)s',

   412             {'et': et.eid, 'wf': wf.eid}, 'et', ask_confirm=False)

   413         rql('SET WF initial_state S WHERE ET initial_state S, ET eid %(et)s, WF eid %(wf)s',

   414             {'et': et.eid, 'wf': wf.eid}, 'et', ask_confirm=False)

   415 

   416 

   417     rql('DELETE TrInfo TI WHERE NOT TI from_state S')

   418     rql('SET TI by_transition T WHERE TI from_state FS, TI to_state TS, '

   419         'FS allowed_transition T, T destination_state TS')

   420     commit()

   421 

   422     drop_relation_definition('State', 'state_of', 'CWEType')

   423     drop_relation_definition('Transition', 'transition_of', 'CWEType')

   424     drop_relation_definition('CWEType', 'initial_state', 'State')

   425 

   426     sync_schema_props_perms()

   427 

   428 if applcubicwebversion < (3, 2, 2) and cubicwebversion >= (3, 2, 1):

   429     from base64 import b64encode

   430     for eid, extid in sql('SELECT eid, extid FROM entities '

   431                           'WHERE extid is NOT NULL',

   432                           ask_confirm=False):

   433         sql('UPDATE entities SET extid=%(extid)s WHERE eid=%(eid)s',

   434             {'extid': b64encode(extid), 'eid': eid}, ask_confirm=False)

   435     commit()

   436 

   437 if applcubicwebversion < (3, 2, 0) and cubicwebversion >= (3, 2, 0):

   438     add_cube('card', update_database=False)

   439 

   440 

   441 if applcubicwebversion < (3, 21, 1) and cubicwebversion >= (3, 21, 1):

   442     add_relation_definition('CWComputedRType', 'read_permission', 'CWGroup')

   443     add_relation_definition('CWComputedRType', 'read_permission', 'RQLExpression')

   444 

   445 

   446 def sync_constraint_types():

   447     """Make sure the repository knows about all constraint types defined in the code"""

   448     from cubicweb.schema import CONSTRAINTS

   449     repo_constraints = set(row[0] for row in rql('Any N WHERE X is CWConstraintType, X name N'))

   450 

   451     for cstrtype in set(CONSTRAINTS) - repo_constraints:

   452         if cstrtype == 'BoundConstraint':

   453             # was renamed to BoundaryConstraint, we don't need the old name

   454             continue

   455         rql('INSERT CWConstraintType X: X name %(name)s', {'name': cstrtype})

   456 

   457     commit()

   458 

   459 sync_constraint_types()