# HG changeset patch # User Erica Marco # Date 1426603840 -3600 # Node ID 1ae64186af9448dffbeebdef910c8c7391c04313 # Parent d5e298df98d105919375afc62e025cdbace36421 [migration] reorder system tables / meta-schema migration into bootstrap_migration Also fix a bug in the backwards-compat eid_type_source method where it returned an extid (possibly null) instead of a source name. With these changes, migrating a database from cw 3.12 is now possible. Closes #4962681 diff -r d5e298df98d1 -r 1ae64186af94 misc/migration/3.14.4_Any.py --- a/misc/migration/3.14.4_Any.py Wed Mar 18 12:02:09 2015 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,10 +0,0 @@ -from yams import schema2sql as y2sql - -dbhelper = repo.system_source.dbhelper -rdefdef = schema['CWSource'].rdef('name') -attrtype = y2sql.type_from_constraints(dbhelper, rdefdef.object, rdefdef.constraints).split()[0] - -cursor = session.cnxset.cu -sql('UPDATE entities SET asource = source WHERE asource is NULL') -dbhelper.change_col_type(cursor, 'entities', 'asource', attrtype, False) -dbhelper.change_col_type(cursor, 'entities', 'source', attrtype, False) diff -r d5e298df98d1 -r 1ae64186af94 misc/migration/3.18.0_Any.py --- a/misc/migration/3.18.0_Any.py Wed Mar 18 12:02:09 2015 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,151 +0,0 @@ -driver = config.system_source_config['db-driver'] -if not (driver == 'postgres' or driver.startswith('sqlserver')): - import sys - print >>sys.stderr, 'This migration is not supported for backends other than sqlserver or postgres (yet).' - sys.exit(1) - -add_relation_definition('CWAttribute', 'add_permission', 'CWGroup') -add_relation_definition('CWAttribute', 'add_permission', 'RQLExpression') - -# a bad defaultval in 3.13.8 schema was fixed in 3.13.9, but the migration was missed -rql('SET ATTR defaultval NULL WHERE ATTR from_entity E, E name "CWSource", ATTR relation_type T, T name "in_synchronization"') - -# the migration gets confused when we change rdefs out from under it. So -# explicitly remove this size constraint so it doesn't stick around and break -# things later. -rdefeid = schema['defaultval'].rdefs.values()[0].eid -rql('DELETE CWConstraint C WHERE C cstrtype T, T name "SizeConstraint", R constrained_by C, R eid %(eid)s', {'eid': rdefeid}) - -sync_schema_props_perms('defaultval') - -def convert_defaultval(cwattr, default): - from decimal import Decimal - import yams - from cubicweb import Binary - if default is None: - return - if isinstance(default, Binary): - # partially migrated instance, try to be idempotent - return default - atype = cwattr.to_entity[0].name - if atype == 'Boolean': - # boolean attributes with default=False were stored as '' - assert default in ('True', 'False', ''), repr(default) - default = default == 'True' - elif atype in ('Int', 'BigInt'): - default = int(default) - elif atype == 'Float': - default = float(default) - elif atype == 'Decimal': - default = Decimal(default) - elif atype in ('Date', 'Datetime', 'TZDatetime', 'Time'): - try: - # handle NOW and TODAY, keep them stored as strings - yams.KEYWORD_MAP[atype][default.upper()] - default = default.upper() - except KeyError: - # otherwise get an actual date or datetime - default = yams.DATE_FACTORY_MAP[atype](default) - else: - assert atype == 'String', atype - default = unicode(default) - return Binary.zpickle(default) - -dbh = repo.system_source.dbhelper - - -sql('ALTER TABLE cw_cwattribute ADD new_defaultval %s' % dbh.TYPE_MAPPING['Bytes']) - -for cwattr in rql('CWAttribute X').entities(): - olddefault = cwattr.defaultval - if olddefault is not None: - req = "UPDATE cw_cwattribute SET new_defaultval = %(val)s WHERE cw_eid = %(eid)s" - args = {'val': dbh.binary_value(convert_defaultval(cwattr, olddefault).getvalue()), 'eid': cwattr.eid} - sql(req, args, ask_confirm=False) - -sql('ALTER TABLE cw_cwattribute DROP COLUMN cw_defaultval') -if driver == 'postgres': - sql('ALTER TABLE cw_cwattribute RENAME COLUMN new_defaultval TO cw_defaultval') -else: # sqlserver - sql("sp_rename 'cw_cwattribute.new_defaultval', 'cw_defaultval', 'COLUMN'") - - -# Set object type to "Bytes" for CWAttribute's "defaultval" attribute -rql('SET X to_entity B WHERE X is CWAttribute, X from_entity Y, Y name "CWAttribute", ' - 'X relation_type Z, Z name "defaultval", B name "Bytes", NOT X to_entity B') - -oldrdef = schema['CWAttribute'].rdef('defaultval') -import yams.buildobjs as ybo -newrdef = ybo.RelationDefinition('CWAttribute', 'defaultval', 'Bytes') -newrdef.eid = oldrdef.eid -schema.add_relation_def(newrdef) -schema.del_relation_def('CWAttribute', 'defaultval', 'String') - -commit() - -sync_schema_props_perms('defaultval') - -for rschema in schema.relations(): - if rschema.symmetric: - subjects = set(repr(e.type) for e in rschema.subjects()) - objects = set(repr(e.type) for e in rschema.objects()) - assert subjects == objects - martians = set(str(eid) for eid, in sql('SELECT eid_to FROM %s_relation, entities WHERE eid_to = eid AND type NOT IN (%s)' % - (rschema.type, ','.join(subjects)))) - martians |= set(str(eid) for eid, in sql('SELECT eid_from FROM %s_relation, entities WHERE eid_from = eid AND type NOT IN (%s)' % - (rschema.type, ','.join(subjects)))) - if martians: - martians = ','.join(martians) - print 'deleting broken relations %s for eids %s' % (rschema.type, martians) - sql('DELETE FROM %s_relation WHERE eid_from IN (%s) OR eid_to IN (%s)' % (rschema.type, martians, martians)) - with session.deny_all_hooks_but(): - rql('SET X %(r)s Y WHERE Y %(r)s X, NOT X %(r)s Y' % {'r': rschema.type}) - commit() - - -# multi columns unique constraints regeneration -from cubicweb.server import schemaserial - -# syncschema hooks would try to remove indices but -# 1) we already do that below -# 2) the hook expects the CWUniqueTogetherConstraint.name attribute that hasn't -# yet been added -with session.allow_all_hooks_but('syncschema'): - rql('DELETE CWUniqueTogetherConstraint C') -commit() - -add_attribute('CWUniqueTogetherConstraint', 'name') - -# low-level wipe code for postgres & sqlserver, plain sql ... -if driver == 'postgres': - for indexname, in sql('select indexname from pg_indexes'): - if indexname.startswith('unique_'): - print 'dropping index', indexname - sql('DROP INDEX %s' % indexname) - commit() -elif driver.startswith('sqlserver'): - for viewname, in sql('select name from sys.views'): - if viewname.startswith('utv_'): - print 'dropping view (index should be cascade-deleted)', viewname - sql('DROP VIEW %s' % viewname) - commit() - -# recreate the constraints, hook will lead to low-level recreation -for eschema in sorted(schema.entities()): - if eschema._unique_together: - print 'recreate unique indexes for', eschema - rql_args = schemaserial.uniquetogether2rqls(eschema) - for rql, args in rql_args: - args['x'] = eschema.eid - session.execute(rql, args) -commit() - -# all attributes perms have to be refreshed ... -for rschema in sorted(schema.relations()): - if rschema.final: - if rschema.type in fsschema: - print 'sync perms for', rschema.type - sync_schema_props_perms(rschema.type, syncprops=False, ask_confirm=False, commit=False) - else: - print 'WARNING: attribute %s missing from fs schema' % rschema.type -commit() diff -r d5e298df98d1 -r 1ae64186af94 misc/migration/bootstrapmigration_repository.py --- a/misc/migration/bootstrapmigration_repository.py Wed Mar 18 12:02:09 2015 +0100 +++ b/misc/migration/bootstrapmigration_repository.py Tue Mar 17 15:50:40 2015 +0100 @@ -49,6 +49,27 @@ cursor.execute(dbh.sql_restart_numrange('entities_id_seq', initial_value=lasteid)) session.commit() +if applcubicwebversion <= (3, 13, 0) and cubicwebversion >= (3, 13, 1): + sql('ALTER TABLE entities ADD asource VARCHAR(64)') + sql('UPDATE entities SET asource=cw_name ' + 'FROM cw_CWSource, cw_source_relation ' + 'WHERE entities.eid=cw_source_relation.eid_from AND cw_source_relation.eid_to=cw_CWSource.cw_eid') + commit() + +if applcubicwebversion <= (3, 14, 4) and cubicwebversion >= (3, 14, 4): + from yams import schema2sql as y2sql + dbhelper = repo.system_source.dbhelper + rdefdef = schema['CWSource'].rdef('name') + attrtype = y2sql.type_from_constraints(dbhelper, rdefdef.object, rdefdef.constraints).split()[0] + cursor = session.cnxset.cu + sql('UPDATE entities SET asource = source WHERE asource is NULL') + dbhelper.change_col_type(cursor, 'entities', 'asource', attrtype, False) + dbhelper.change_col_type(cursor, 'entities', 'source', attrtype, False) + + # we now have a functional asource column, start using the normal eid_type_source method + if repo.system_source.eid_type_source == repo.system_source.eid_type_source_pre_131: + del repo.system_source.eid_type_source + if applcubicwebversion < (3, 19, 0) and cubicwebversion >= (3, 19, 0): try: # need explicit drop of the indexes on some database systems (sqlserver) @@ -63,6 +84,163 @@ replace_eid_sequence_with_eid_numrange(session) +if schema['TZDatetime'].eid is None: + add_entity_type('TZDatetime', auto=False) +if schema['TZTime'].eid is None: + add_entity_type('TZTime', auto=False) + +if applcubicwebversion < (3, 18, 0) and cubicwebversion >= (3, 18, 0): + driver = config.system_source_config['db-driver'] + if not (driver == 'postgres' or driver.startswith('sqlserver')): + import sys + print >>sys.stderr, 'This migration is not supported for backends other than sqlserver or postgres (yet).' + sys.exit(1) + + add_relation_definition('CWAttribute', 'add_permission', 'CWGroup') + add_relation_definition('CWAttribute', 'add_permission', 'RQLExpression') + + # a bad defaultval in 3.13.8 schema was fixed in 3.13.9, but the migration was missed + rql('SET ATTR defaultval NULL WHERE ATTR from_entity E, E name "CWSource", ATTR relation_type T, T name "in_synchronization"') + + # the migration gets confused when we change rdefs out from under it. So + # explicitly remove this size constraint so it doesn't stick around and break + # things later. + rdefeid = schema['defaultval'].rdefs.values()[0].eid + rql('DELETE CWConstraint C WHERE C cstrtype T, T name "SizeConstraint", R constrained_by C, R eid %(eid)s', {'eid': rdefeid}) + + sync_schema_props_perms('defaultval') + + def convert_defaultval(cwattr, default): + from decimal import Decimal + import yams + from cubicweb import Binary + if default is None: + return + if isinstance(default, Binary): + # partially migrated instance, try to be idempotent + return default + atype = cwattr.to_entity[0].name + if atype == 'Boolean': + # boolean attributes with default=False were stored as '' + assert default in ('True', 'False', ''), repr(default) + default = default == 'True' + elif atype in ('Int', 'BigInt'): + default = int(default) + elif atype == 'Float': + default = float(default) + elif atype == 'Decimal': + default = Decimal(default) + elif atype in ('Date', 'Datetime', 'TZDatetime', 'Time'): + try: + # handle NOW and TODAY, keep them stored as strings + yams.KEYWORD_MAP[atype][default.upper()] + default = default.upper() + except KeyError: + # otherwise get an actual date or datetime + default = yams.DATE_FACTORY_MAP[atype](default) + else: + assert atype == 'String', atype + default = unicode(default) + return Binary.zpickle(default) + + dbh = repo.system_source.dbhelper + + + sql('ALTER TABLE cw_cwattribute ADD new_defaultval %s' % dbh.TYPE_MAPPING['Bytes']) + + for cwattr in rql('CWAttribute X').entities(): + olddefault = cwattr.defaultval + if olddefault is not None: + req = "UPDATE cw_cwattribute SET new_defaultval = %(val)s WHERE cw_eid = %(eid)s" + args = {'val': dbh.binary_value(convert_defaultval(cwattr, olddefault).getvalue()), 'eid': cwattr.eid} + sql(req, args, ask_confirm=False) + + sql('ALTER TABLE cw_cwattribute DROP COLUMN cw_defaultval') + if driver == 'postgres': + sql('ALTER TABLE cw_cwattribute RENAME COLUMN new_defaultval TO cw_defaultval') + else: # sqlserver + sql("sp_rename 'cw_cwattribute.new_defaultval', 'cw_defaultval', 'COLUMN'") + + + # Set object type to "Bytes" for CWAttribute's "defaultval" attribute + rql('SET X to_entity B WHERE X is CWAttribute, X from_entity Y, Y name "CWAttribute", ' + 'X relation_type Z, Z name "defaultval", B name "Bytes", NOT X to_entity B') + + oldrdef = schema['CWAttribute'].rdef('defaultval') + import yams.buildobjs as ybo + newrdef = ybo.RelationDefinition('CWAttribute', 'defaultval', 'Bytes') + newrdef.eid = oldrdef.eid + schema.add_relation_def(newrdef) + schema.del_relation_def('CWAttribute', 'defaultval', 'String') + + commit() + + sync_schema_props_perms('defaultval') + + for rschema in schema.relations(): + if rschema.symmetric: + subjects = set(repr(e.type) for e in rschema.subjects()) + objects = set(repr(e.type) for e in rschema.objects()) + assert subjects == objects + martians = set(str(eid) for eid, in sql('SELECT eid_to FROM %s_relation, entities WHERE eid_to = eid AND type NOT IN (%s)' % + (rschema.type, ','.join(subjects)))) + martians |= set(str(eid) for eid, in sql('SELECT eid_from FROM %s_relation, entities WHERE eid_from = eid AND type NOT IN (%s)' % + (rschema.type, ','.join(subjects)))) + if martians: + martians = ','.join(martians) + print 'deleting broken relations %s for eids %s' % (rschema.type, martians) + sql('DELETE FROM %s_relation WHERE eid_from IN (%s) OR eid_to IN (%s)' % (rschema.type, martians, martians)) + with session.deny_all_hooks_but(): + rql('SET X %(r)s Y WHERE Y %(r)s X, NOT X %(r)s Y' % {'r': rschema.type}) + commit() + + + # multi columns unique constraints regeneration + from cubicweb.server import schemaserial + + # syncschema hooks would try to remove indices but + # 1) we already do that below + # 2) the hook expects the CWUniqueTogetherConstraint.name attribute that hasn't + # yet been added + with session.allow_all_hooks_but('syncschema'): + rql('DELETE CWUniqueTogetherConstraint C') + commit() + add_attribute('CWUniqueTogetherConstraint', 'name') + + # low-level wipe code for postgres & sqlserver, plain sql ... + if driver == 'postgres': + for indexname, in sql('select indexname from pg_indexes'): + if indexname.startswith('unique_'): + print 'dropping index', indexname + sql('DROP INDEX %s' % indexname) + commit() + elif driver.startswith('sqlserver'): + for viewname, in sql('select name from sys.views'): + if viewname.startswith('utv_'): + print 'dropping view (index should be cascade-deleted)', viewname + sql('DROP VIEW %s' % viewname) + commit() + + # recreate the constraints, hook will lead to low-level recreation + for eschema in sorted(schema.entities()): + if eschema._unique_together: + print 'recreate unique indexes for', eschema + rql_args = schemaserial.uniquetogether2rqls(eschema) + for rql, args in rql_args: + args['x'] = eschema.eid + session.execute(rql, args) + commit() + + # all attributes perms have to be refreshed ... + for rschema in sorted(schema.relations()): + if rschema.final: + if rschema.type in fsschema: + print 'sync perms for', rschema.type + sync_schema_props_perms(rschema.type, syncprops=False, ask_confirm=False, commit=False) + else: + print 'WARNING: attribute %s missing from fs schema' % rschema.type + commit() + if applcubicwebversion < (3, 17, 0) and cubicwebversion >= (3, 17, 0): try: add_cube('sioc', update_database=False) @@ -83,18 +261,6 @@ 'cube, which is not installed. Continue anyway?'): raise -if applcubicwebversion <= (3, 13, 0) and cubicwebversion >= (3, 13, 1): - sql('ALTER TABLE entities ADD asource VARCHAR(64)') - sql('UPDATE entities SET asource=cw_name ' - 'FROM cw_CWSource, cw_source_relation ' - 'WHERE entities.eid=cw_source_relation.eid_from AND cw_source_relation.eid_to=cw_CWSource.cw_eid') - commit() - -if schema['TZDatetime'].eid is None: - add_entity_type('TZDatetime', auto=False) -if schema['TZTime'].eid is None: - add_entity_type('TZTime', auto=False) - if applcubicwebversion <= (3, 14, 0) and cubicwebversion >= (3, 14, 0): if 'require_permission' in schema and not 'localperms'in repo.config.cubes(): @@ -105,14 +271,13 @@ raise ExecutionError('In cubicweb 3.14, CWPermission and related stuff ' 'has been moved to cube localperms. Install it first.') + if applcubicwebversion == (3, 6, 0) and cubicwebversion >= (3, 6, 0): CSTRMAP = dict(rql('Any T, X WHERE X is CWConstraintType, X name T', ask_confirm=False)) _add_relation_definition_no_perms('CWAttribute', 'update_permission', 'CWGroup') _add_relation_definition_no_perms('CWAttribute', 'update_permission', 'RQLExpression') rql('SET X update_permission Y WHERE X is CWAttribute, X add_permission Y') - drop_relation_definition('CWAttribute', 'add_permission', 'CWGroup') - drop_relation_definition('CWAttribute', 'add_permission', 'RQLExpression') drop_relation_definition('CWAttribute', 'delete_permission', 'CWGroup') drop_relation_definition('CWAttribute', 'delete_permission', 'RQLExpression') diff -r d5e298df98d1 -r 1ae64186af94 server/sources/native.py --- a/server/sources/native.py Wed Mar 18 12:02:09 2015 +0100 +++ b/server/sources/native.py Tue Mar 17 15:50:40 2015 +0100 @@ -850,7 +850,7 @@ res = list(res) if res[-1] is not None: res[-1] = b64decode(res[-1]) - res.append(res[1]) + res.append("system") return res def extid2eid(self, cnx, extid):