misc/migration/bootstrapmigration_repository.py
changeset 10270 1ae64186af94
parent 10058 d47c318f95b2
child 10272 3231fd2fa7a5
--- a/misc/migration/bootstrapmigration_repository.py	Wed Mar 18 12:02:09 2015 +0100
+++ b/misc/migration/bootstrapmigration_repository.py	Tue Mar 17 15:50:40 2015 +0100
@@ -49,6 +49,27 @@
     cursor.execute(dbh.sql_restart_numrange('entities_id_seq', initial_value=lasteid))
     session.commit()
 
+if applcubicwebversion <= (3, 13, 0) and cubicwebversion >= (3, 13, 1):
+    sql('ALTER TABLE entities ADD asource VARCHAR(64)')
+    sql('UPDATE entities SET asource=cw_name  '
+        'FROM cw_CWSource, cw_source_relation '
+        'WHERE entities.eid=cw_source_relation.eid_from AND cw_source_relation.eid_to=cw_CWSource.cw_eid')
+    commit()
+
+if applcubicwebversion <= (3, 14, 4) and cubicwebversion >= (3, 14, 4):
+    from yams import schema2sql as y2sql
+    dbhelper = repo.system_source.dbhelper
+    rdefdef = schema['CWSource'].rdef('name')
+    attrtype = y2sql.type_from_constraints(dbhelper, rdefdef.object, rdefdef.constraints).split()[0]
+    cursor = session.cnxset.cu
+    sql('UPDATE entities SET asource = source WHERE asource is NULL')
+    dbhelper.change_col_type(cursor, 'entities', 'asource', attrtype, False)
+    dbhelper.change_col_type(cursor, 'entities', 'source', attrtype, False)
+
+    # we now have a functional asource column, start using the normal eid_type_source method
+    if repo.system_source.eid_type_source == repo.system_source.eid_type_source_pre_131:
+        del repo.system_source.eid_type_source
+
 if applcubicwebversion < (3, 19, 0) and cubicwebversion >= (3, 19, 0):
     try: 
         # need explicit drop of the indexes on some database systems (sqlserver)
@@ -63,6 +84,163 @@
 
     replace_eid_sequence_with_eid_numrange(session)
 
+if schema['TZDatetime'].eid is None:
+    add_entity_type('TZDatetime', auto=False)
+if schema['TZTime'].eid is None:
+    add_entity_type('TZTime', auto=False)
+
+if applcubicwebversion < (3, 18, 0) and cubicwebversion >= (3, 18, 0):
+    driver = config.system_source_config['db-driver']
+    if not (driver == 'postgres' or driver.startswith('sqlserver')):
+        import sys
+        print >>sys.stderr, 'This migration is not supported for backends other than sqlserver or postgres (yet).'
+        sys.exit(1)
+
+    add_relation_definition('CWAttribute', 'add_permission', 'CWGroup')
+    add_relation_definition('CWAttribute', 'add_permission', 'RQLExpression')
+
+    # a bad defaultval in 3.13.8 schema was fixed in 3.13.9, but the migration was missed
+    rql('SET ATTR defaultval NULL WHERE ATTR from_entity E, E name "CWSource", ATTR relation_type T, T name "in_synchronization"')
+
+    # the migration gets confused when we change rdefs out from under it.  So
+    # explicitly remove this size constraint so it doesn't stick around and break
+    # things later.
+    rdefeid = schema['defaultval'].rdefs.values()[0].eid
+    rql('DELETE CWConstraint C WHERE C cstrtype T, T name "SizeConstraint", R constrained_by C, R eid %(eid)s', {'eid': rdefeid})
+
+    sync_schema_props_perms('defaultval')
+
+    def convert_defaultval(cwattr, default):
+        from decimal import Decimal
+        import yams
+        from cubicweb import Binary
+        if default is None:
+            return
+        if isinstance(default, Binary):
+            # partially migrated instance, try to be idempotent
+            return default
+        atype = cwattr.to_entity[0].name
+        if atype == 'Boolean':
+            # boolean attributes with default=False were stored as ''
+            assert default in ('True', 'False', ''), repr(default)
+            default = default == 'True'
+        elif atype in ('Int', 'BigInt'):
+            default = int(default)
+        elif atype == 'Float':
+            default = float(default)
+        elif atype == 'Decimal':
+            default = Decimal(default)
+        elif atype in ('Date', 'Datetime', 'TZDatetime', 'Time'):
+            try:
+                # handle NOW and TODAY, keep them stored as strings
+                yams.KEYWORD_MAP[atype][default.upper()]
+                default = default.upper()
+            except KeyError:
+                # otherwise get an actual date or datetime
+                default = yams.DATE_FACTORY_MAP[atype](default)
+        else:
+            assert atype == 'String', atype
+            default = unicode(default)
+        return Binary.zpickle(default)
+
+    dbh = repo.system_source.dbhelper
+
+
+    sql('ALTER TABLE cw_cwattribute ADD new_defaultval %s' % dbh.TYPE_MAPPING['Bytes'])
+
+    for cwattr in rql('CWAttribute X').entities():
+        olddefault = cwattr.defaultval
+        if olddefault is not None:
+            req = "UPDATE cw_cwattribute SET new_defaultval = %(val)s WHERE cw_eid = %(eid)s"
+            args = {'val': dbh.binary_value(convert_defaultval(cwattr, olddefault).getvalue()), 'eid': cwattr.eid}
+            sql(req, args, ask_confirm=False)
+
+    sql('ALTER TABLE cw_cwattribute DROP COLUMN cw_defaultval')
+    if driver == 'postgres':
+        sql('ALTER TABLE cw_cwattribute RENAME COLUMN new_defaultval TO cw_defaultval')
+    else: # sqlserver
+        sql("sp_rename 'cw_cwattribute.new_defaultval', 'cw_defaultval', 'COLUMN'")
+
+
+    # Set object type to "Bytes" for CWAttribute's "defaultval" attribute
+    rql('SET X to_entity B WHERE X is CWAttribute, X from_entity Y, Y name "CWAttribute", '
+        'X relation_type Z, Z name "defaultval", B name "Bytes", NOT X to_entity B')
+
+    oldrdef = schema['CWAttribute'].rdef('defaultval')
+    import yams.buildobjs as ybo
+    newrdef = ybo.RelationDefinition('CWAttribute', 'defaultval', 'Bytes')
+    newrdef.eid = oldrdef.eid
+    schema.add_relation_def(newrdef)
+    schema.del_relation_def('CWAttribute', 'defaultval', 'String')
+
+    commit()
+
+    sync_schema_props_perms('defaultval')
+
+    for rschema in schema.relations():
+        if rschema.symmetric:
+            subjects = set(repr(e.type) for e in rschema.subjects())
+            objects = set(repr(e.type) for e in rschema.objects())
+            assert subjects == objects
+            martians = set(str(eid) for eid, in sql('SELECT eid_to FROM %s_relation, entities WHERE eid_to = eid AND type NOT IN (%s)' %
+                                               (rschema.type, ','.join(subjects))))
+            martians |= set(str(eid) for eid, in sql('SELECT eid_from FROM %s_relation, entities WHERE eid_from = eid AND type NOT IN (%s)' %
+                                                (rschema.type, ','.join(subjects))))
+            if martians:
+                martians = ','.join(martians)
+                print 'deleting broken relations %s for eids %s' % (rschema.type, martians)
+                sql('DELETE FROM %s_relation WHERE eid_from IN (%s) OR eid_to IN (%s)' % (rschema.type, martians, martians))
+            with session.deny_all_hooks_but():
+                rql('SET X %(r)s Y WHERE Y %(r)s X, NOT X %(r)s Y' % {'r': rschema.type})
+            commit()
+
+
+    # multi columns unique constraints regeneration
+    from cubicweb.server import schemaserial
+
+    # syncschema hooks would try to remove indices but
+    # 1) we already do that below
+    # 2) the hook expects the CWUniqueTogetherConstraint.name attribute that hasn't
+    #    yet been added
+    with session.allow_all_hooks_but('syncschema'):
+        rql('DELETE CWUniqueTogetherConstraint C')
+    commit()
+    add_attribute('CWUniqueTogetherConstraint', 'name')
+
+    # low-level wipe code for postgres & sqlserver, plain sql ...
+    if driver == 'postgres':
+        for indexname, in sql('select indexname from pg_indexes'):
+            if indexname.startswith('unique_'):
+                print 'dropping index', indexname
+                sql('DROP INDEX %s' % indexname)
+        commit()
+    elif driver.startswith('sqlserver'):
+        for viewname, in sql('select name from sys.views'):
+            if viewname.startswith('utv_'):
+                print 'dropping view (index should be cascade-deleted)', viewname
+                sql('DROP VIEW %s' % viewname)
+        commit()
+
+    # recreate the constraints, hook will lead to low-level recreation
+    for eschema in sorted(schema.entities()):
+        if eschema._unique_together:
+            print 'recreate unique indexes for', eschema
+            rql_args = schemaserial.uniquetogether2rqls(eschema)
+            for rql, args in rql_args:
+                args['x'] = eschema.eid
+                session.execute(rql, args)
+    commit()
+
+    # all attributes perms have to be refreshed ...
+    for rschema in sorted(schema.relations()):
+        if rschema.final:
+            if rschema.type in fsschema:
+                print 'sync perms for', rschema.type
+                sync_schema_props_perms(rschema.type, syncprops=False, ask_confirm=False, commit=False)
+            else:
+                print 'WARNING: attribute %s missing from fs schema' % rschema.type
+    commit()
+
 if applcubicwebversion < (3, 17, 0) and cubicwebversion >= (3, 17, 0):
     try:
         add_cube('sioc', update_database=False)
@@ -83,18 +261,6 @@
                        'cube, which is not installed.  Continue anyway?'):
             raise
 
-if applcubicwebversion <= (3, 13, 0) and cubicwebversion >= (3, 13, 1):
-    sql('ALTER TABLE entities ADD asource VARCHAR(64)')
-    sql('UPDATE entities SET asource=cw_name  '
-        'FROM cw_CWSource, cw_source_relation '
-        'WHERE entities.eid=cw_source_relation.eid_from AND cw_source_relation.eid_to=cw_CWSource.cw_eid')
-    commit()
-
-if schema['TZDatetime'].eid is None:
-    add_entity_type('TZDatetime', auto=False)
-if schema['TZTime'].eid is None:
-    add_entity_type('TZTime', auto=False)
-
 
 if applcubicwebversion <= (3, 14, 0) and cubicwebversion >= (3, 14, 0):
     if 'require_permission' in schema and not 'localperms'in repo.config.cubes():
@@ -105,14 +271,13 @@
             raise ExecutionError('In cubicweb 3.14, CWPermission and related stuff '
                                  'has been moved to cube localperms. Install it first.')
 
+
 if applcubicwebversion == (3, 6, 0) and cubicwebversion >= (3, 6, 0):
     CSTRMAP = dict(rql('Any T, X WHERE X is CWConstraintType, X name T',
                        ask_confirm=False))
     _add_relation_definition_no_perms('CWAttribute', 'update_permission', 'CWGroup')
     _add_relation_definition_no_perms('CWAttribute', 'update_permission', 'RQLExpression')
     rql('SET X update_permission Y WHERE X is CWAttribute, X add_permission Y')
-    drop_relation_definition('CWAttribute', 'add_permission', 'CWGroup')
-    drop_relation_definition('CWAttribute', 'add_permission', 'RQLExpression')
     drop_relation_definition('CWAttribute', 'delete_permission', 'CWGroup')
     drop_relation_definition('CWAttribute', 'delete_permission', 'RQLExpression')