[migration] reorder system tables / meta-schema migration into bootstrap_migration
Also fix a bug in the backwards-compat eid_type_source method where it
returned an extid (possibly null) instead of a source name.
With these changes, migrating a database from cw 3.12 is now possible.
Closes #4962681
--- a/misc/migration/3.14.4_Any.py Wed Mar 18 12:02:09 2015 +0100
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,10 +0,0 @@
-from yams import schema2sql as y2sql
-
-dbhelper = repo.system_source.dbhelper
-rdefdef = schema['CWSource'].rdef('name')
-attrtype = y2sql.type_from_constraints(dbhelper, rdefdef.object, rdefdef.constraints).split()[0]
-
-cursor = session.cnxset.cu
-sql('UPDATE entities SET asource = source WHERE asource is NULL')
-dbhelper.change_col_type(cursor, 'entities', 'asource', attrtype, False)
-dbhelper.change_col_type(cursor, 'entities', 'source', attrtype, False)
--- a/misc/migration/3.18.0_Any.py Wed Mar 18 12:02:09 2015 +0100
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,151 +0,0 @@
-driver = config.system_source_config['db-driver']
-if not (driver == 'postgres' or driver.startswith('sqlserver')):
- import sys
- print >>sys.stderr, 'This migration is not supported for backends other than sqlserver or postgres (yet).'
- sys.exit(1)
-
-add_relation_definition('CWAttribute', 'add_permission', 'CWGroup')
-add_relation_definition('CWAttribute', 'add_permission', 'RQLExpression')
-
-# a bad defaultval in 3.13.8 schema was fixed in 3.13.9, but the migration was missed
-rql('SET ATTR defaultval NULL WHERE ATTR from_entity E, E name "CWSource", ATTR relation_type T, T name "in_synchronization"')
-
-# the migration gets confused when we change rdefs out from under it. So
-# explicitly remove this size constraint so it doesn't stick around and break
-# things later.
-rdefeid = schema['defaultval'].rdefs.values()[0].eid
-rql('DELETE CWConstraint C WHERE C cstrtype T, T name "SizeConstraint", R constrained_by C, R eid %(eid)s', {'eid': rdefeid})
-
-sync_schema_props_perms('defaultval')
-
-def convert_defaultval(cwattr, default):
- from decimal import Decimal
- import yams
- from cubicweb import Binary
- if default is None:
- return
- if isinstance(default, Binary):
- # partially migrated instance, try to be idempotent
- return default
- atype = cwattr.to_entity[0].name
- if atype == 'Boolean':
- # boolean attributes with default=False were stored as ''
- assert default in ('True', 'False', ''), repr(default)
- default = default == 'True'
- elif atype in ('Int', 'BigInt'):
- default = int(default)
- elif atype == 'Float':
- default = float(default)
- elif atype == 'Decimal':
- default = Decimal(default)
- elif atype in ('Date', 'Datetime', 'TZDatetime', 'Time'):
- try:
- # handle NOW and TODAY, keep them stored as strings
- yams.KEYWORD_MAP[atype][default.upper()]
- default = default.upper()
- except KeyError:
- # otherwise get an actual date or datetime
- default = yams.DATE_FACTORY_MAP[atype](default)
- else:
- assert atype == 'String', atype
- default = unicode(default)
- return Binary.zpickle(default)
-
-dbh = repo.system_source.dbhelper
-
-
-sql('ALTER TABLE cw_cwattribute ADD new_defaultval %s' % dbh.TYPE_MAPPING['Bytes'])
-
-for cwattr in rql('CWAttribute X').entities():
- olddefault = cwattr.defaultval
- if olddefault is not None:
- req = "UPDATE cw_cwattribute SET new_defaultval = %(val)s WHERE cw_eid = %(eid)s"
- args = {'val': dbh.binary_value(convert_defaultval(cwattr, olddefault).getvalue()), 'eid': cwattr.eid}
- sql(req, args, ask_confirm=False)
-
-sql('ALTER TABLE cw_cwattribute DROP COLUMN cw_defaultval')
-if driver == 'postgres':
- sql('ALTER TABLE cw_cwattribute RENAME COLUMN new_defaultval TO cw_defaultval')
-else: # sqlserver
- sql("sp_rename 'cw_cwattribute.new_defaultval', 'cw_defaultval', 'COLUMN'")
-
-
-# Set object type to "Bytes" for CWAttribute's "defaultval" attribute
-rql('SET X to_entity B WHERE X is CWAttribute, X from_entity Y, Y name "CWAttribute", '
- 'X relation_type Z, Z name "defaultval", B name "Bytes", NOT X to_entity B')
-
-oldrdef = schema['CWAttribute'].rdef('defaultval')
-import yams.buildobjs as ybo
-newrdef = ybo.RelationDefinition('CWAttribute', 'defaultval', 'Bytes')
-newrdef.eid = oldrdef.eid
-schema.add_relation_def(newrdef)
-schema.del_relation_def('CWAttribute', 'defaultval', 'String')
-
-commit()
-
-sync_schema_props_perms('defaultval')
-
-for rschema in schema.relations():
- if rschema.symmetric:
- subjects = set(repr(e.type) for e in rschema.subjects())
- objects = set(repr(e.type) for e in rschema.objects())
- assert subjects == objects
- martians = set(str(eid) for eid, in sql('SELECT eid_to FROM %s_relation, entities WHERE eid_to = eid AND type NOT IN (%s)' %
- (rschema.type, ','.join(subjects))))
- martians |= set(str(eid) for eid, in sql('SELECT eid_from FROM %s_relation, entities WHERE eid_from = eid AND type NOT IN (%s)' %
- (rschema.type, ','.join(subjects))))
- if martians:
- martians = ','.join(martians)
- print 'deleting broken relations %s for eids %s' % (rschema.type, martians)
- sql('DELETE FROM %s_relation WHERE eid_from IN (%s) OR eid_to IN (%s)' % (rschema.type, martians, martians))
- with session.deny_all_hooks_but():
- rql('SET X %(r)s Y WHERE Y %(r)s X, NOT X %(r)s Y' % {'r': rschema.type})
- commit()
-
-
-# multi columns unique constraints regeneration
-from cubicweb.server import schemaserial
-
-# syncschema hooks would try to remove indices but
-# 1) we already do that below
-# 2) the hook expects the CWUniqueTogetherConstraint.name attribute that hasn't
-# yet been added
-with session.allow_all_hooks_but('syncschema'):
- rql('DELETE CWUniqueTogetherConstraint C')
-commit()
-
-add_attribute('CWUniqueTogetherConstraint', 'name')
-
-# low-level wipe code for postgres & sqlserver, plain sql ...
-if driver == 'postgres':
- for indexname, in sql('select indexname from pg_indexes'):
- if indexname.startswith('unique_'):
- print 'dropping index', indexname
- sql('DROP INDEX %s' % indexname)
- commit()
-elif driver.startswith('sqlserver'):
- for viewname, in sql('select name from sys.views'):
- if viewname.startswith('utv_'):
- print 'dropping view (index should be cascade-deleted)', viewname
- sql('DROP VIEW %s' % viewname)
- commit()
-
-# recreate the constraints, hook will lead to low-level recreation
-for eschema in sorted(schema.entities()):
- if eschema._unique_together:
- print 'recreate unique indexes for', eschema
- rql_args = schemaserial.uniquetogether2rqls(eschema)
- for rql, args in rql_args:
- args['x'] = eschema.eid
- session.execute(rql, args)
-commit()
-
-# all attributes perms have to be refreshed ...
-for rschema in sorted(schema.relations()):
- if rschema.final:
- if rschema.type in fsschema:
- print 'sync perms for', rschema.type
- sync_schema_props_perms(rschema.type, syncprops=False, ask_confirm=False, commit=False)
- else:
- print 'WARNING: attribute %s missing from fs schema' % rschema.type
-commit()
--- a/misc/migration/bootstrapmigration_repository.py Wed Mar 18 12:02:09 2015 +0100
+++ b/misc/migration/bootstrapmigration_repository.py Tue Mar 17 15:50:40 2015 +0100
@@ -49,6 +49,27 @@
cursor.execute(dbh.sql_restart_numrange('entities_id_seq', initial_value=lasteid))
session.commit()
+if applcubicwebversion <= (3, 13, 0) and cubicwebversion >= (3, 13, 1):
+ sql('ALTER TABLE entities ADD asource VARCHAR(64)')
+ sql('UPDATE entities SET asource=cw_name '
+ 'FROM cw_CWSource, cw_source_relation '
+ 'WHERE entities.eid=cw_source_relation.eid_from AND cw_source_relation.eid_to=cw_CWSource.cw_eid')
+ commit()
+
+if applcubicwebversion <= (3, 14, 4) and cubicwebversion >= (3, 14, 4):
+ from yams import schema2sql as y2sql
+ dbhelper = repo.system_source.dbhelper
+ rdefdef = schema['CWSource'].rdef('name')
+ attrtype = y2sql.type_from_constraints(dbhelper, rdefdef.object, rdefdef.constraints).split()[0]
+ cursor = session.cnxset.cu
+ sql('UPDATE entities SET asource = source WHERE asource is NULL')
+ dbhelper.change_col_type(cursor, 'entities', 'asource', attrtype, False)
+ dbhelper.change_col_type(cursor, 'entities', 'source', attrtype, False)
+
+ # we now have a functional asource column, start using the normal eid_type_source method
+ if repo.system_source.eid_type_source == repo.system_source.eid_type_source_pre_131:
+ del repo.system_source.eid_type_source
+
if applcubicwebversion < (3, 19, 0) and cubicwebversion >= (3, 19, 0):
try:
# need explicit drop of the indexes on some database systems (sqlserver)
@@ -63,6 +84,163 @@
replace_eid_sequence_with_eid_numrange(session)
+if schema['TZDatetime'].eid is None:
+ add_entity_type('TZDatetime', auto=False)
+if schema['TZTime'].eid is None:
+ add_entity_type('TZTime', auto=False)
+
+if applcubicwebversion < (3, 18, 0) and cubicwebversion >= (3, 18, 0):
+ driver = config.system_source_config['db-driver']
+ if not (driver == 'postgres' or driver.startswith('sqlserver')):
+ import sys
+ print >>sys.stderr, 'This migration is not supported for backends other than sqlserver or postgres (yet).'
+ sys.exit(1)
+
+ add_relation_definition('CWAttribute', 'add_permission', 'CWGroup')
+ add_relation_definition('CWAttribute', 'add_permission', 'RQLExpression')
+
+ # a bad defaultval in 3.13.8 schema was fixed in 3.13.9, but the migration was missed
+ rql('SET ATTR defaultval NULL WHERE ATTR from_entity E, E name "CWSource", ATTR relation_type T, T name "in_synchronization"')
+
+ # the migration gets confused when we change rdefs out from under it. So
+ # explicitly remove this size constraint so it doesn't stick around and break
+ # things later.
+ rdefeid = schema['defaultval'].rdefs.values()[0].eid
+ rql('DELETE CWConstraint C WHERE C cstrtype T, T name "SizeConstraint", R constrained_by C, R eid %(eid)s', {'eid': rdefeid})
+
+ sync_schema_props_perms('defaultval')
+
+ def convert_defaultval(cwattr, default):
+ from decimal import Decimal
+ import yams
+ from cubicweb import Binary
+ if default is None:
+ return
+ if isinstance(default, Binary):
+ # partially migrated instance, try to be idempotent
+ return default
+ atype = cwattr.to_entity[0].name
+ if atype == 'Boolean':
+ # boolean attributes with default=False were stored as ''
+ assert default in ('True', 'False', ''), repr(default)
+ default = default == 'True'
+ elif atype in ('Int', 'BigInt'):
+ default = int(default)
+ elif atype == 'Float':
+ default = float(default)
+ elif atype == 'Decimal':
+ default = Decimal(default)
+ elif atype in ('Date', 'Datetime', 'TZDatetime', 'Time'):
+ try:
+ # handle NOW and TODAY, keep them stored as strings
+ yams.KEYWORD_MAP[atype][default.upper()]
+ default = default.upper()
+ except KeyError:
+ # otherwise get an actual date or datetime
+ default = yams.DATE_FACTORY_MAP[atype](default)
+ else:
+ assert atype == 'String', atype
+ default = unicode(default)
+ return Binary.zpickle(default)
+
+ dbh = repo.system_source.dbhelper
+
+
+ sql('ALTER TABLE cw_cwattribute ADD new_defaultval %s' % dbh.TYPE_MAPPING['Bytes'])
+
+ for cwattr in rql('CWAttribute X').entities():
+ olddefault = cwattr.defaultval
+ if olddefault is not None:
+ req = "UPDATE cw_cwattribute SET new_defaultval = %(val)s WHERE cw_eid = %(eid)s"
+ args = {'val': dbh.binary_value(convert_defaultval(cwattr, olddefault).getvalue()), 'eid': cwattr.eid}
+ sql(req, args, ask_confirm=False)
+
+ sql('ALTER TABLE cw_cwattribute DROP COLUMN cw_defaultval')
+ if driver == 'postgres':
+ sql('ALTER TABLE cw_cwattribute RENAME COLUMN new_defaultval TO cw_defaultval')
+ else: # sqlserver
+ sql("sp_rename 'cw_cwattribute.new_defaultval', 'cw_defaultval', 'COLUMN'")
+
+
+ # Set object type to "Bytes" for CWAttribute's "defaultval" attribute
+ rql('SET X to_entity B WHERE X is CWAttribute, X from_entity Y, Y name "CWAttribute", '
+ 'X relation_type Z, Z name "defaultval", B name "Bytes", NOT X to_entity B')
+
+ oldrdef = schema['CWAttribute'].rdef('defaultval')
+ import yams.buildobjs as ybo
+ newrdef = ybo.RelationDefinition('CWAttribute', 'defaultval', 'Bytes')
+ newrdef.eid = oldrdef.eid
+ schema.add_relation_def(newrdef)
+ schema.del_relation_def('CWAttribute', 'defaultval', 'String')
+
+ commit()
+
+ sync_schema_props_perms('defaultval')
+
+ for rschema in schema.relations():
+ if rschema.symmetric:
+ subjects = set(repr(e.type) for e in rschema.subjects())
+ objects = set(repr(e.type) for e in rschema.objects())
+ assert subjects == objects
+ martians = set(str(eid) for eid, in sql('SELECT eid_to FROM %s_relation, entities WHERE eid_to = eid AND type NOT IN (%s)' %
+ (rschema.type, ','.join(subjects))))
+ martians |= set(str(eid) for eid, in sql('SELECT eid_from FROM %s_relation, entities WHERE eid_from = eid AND type NOT IN (%s)' %
+ (rschema.type, ','.join(subjects))))
+ if martians:
+ martians = ','.join(martians)
+ print 'deleting broken relations %s for eids %s' % (rschema.type, martians)
+ sql('DELETE FROM %s_relation WHERE eid_from IN (%s) OR eid_to IN (%s)' % (rschema.type, martians, martians))
+ with session.deny_all_hooks_but():
+ rql('SET X %(r)s Y WHERE Y %(r)s X, NOT X %(r)s Y' % {'r': rschema.type})
+ commit()
+
+
+ # multi columns unique constraints regeneration
+ from cubicweb.server import schemaserial
+
+ # syncschema hooks would try to remove indices but
+ # 1) we already do that below
+ # 2) the hook expects the CWUniqueTogetherConstraint.name attribute that hasn't
+ # yet been added
+ with session.allow_all_hooks_but('syncschema'):
+ rql('DELETE CWUniqueTogetherConstraint C')
+ commit()
+ add_attribute('CWUniqueTogetherConstraint', 'name')
+
+ # low-level wipe code for postgres & sqlserver, plain sql ...
+ if driver == 'postgres':
+ for indexname, in sql('select indexname from pg_indexes'):
+ if indexname.startswith('unique_'):
+ print 'dropping index', indexname
+ sql('DROP INDEX %s' % indexname)
+ commit()
+ elif driver.startswith('sqlserver'):
+ for viewname, in sql('select name from sys.views'):
+ if viewname.startswith('utv_'):
+ print 'dropping view (index should be cascade-deleted)', viewname
+ sql('DROP VIEW %s' % viewname)
+ commit()
+
+ # recreate the constraints, hook will lead to low-level recreation
+ for eschema in sorted(schema.entities()):
+ if eschema._unique_together:
+ print 'recreate unique indexes for', eschema
+ rql_args = schemaserial.uniquetogether2rqls(eschema)
+ for rql, args in rql_args:
+ args['x'] = eschema.eid
+ session.execute(rql, args)
+ commit()
+
+ # all attributes perms have to be refreshed ...
+ for rschema in sorted(schema.relations()):
+ if rschema.final:
+ if rschema.type in fsschema:
+ print 'sync perms for', rschema.type
+ sync_schema_props_perms(rschema.type, syncprops=False, ask_confirm=False, commit=False)
+ else:
+ print 'WARNING: attribute %s missing from fs schema' % rschema.type
+ commit()
+
if applcubicwebversion < (3, 17, 0) and cubicwebversion >= (3, 17, 0):
try:
add_cube('sioc', update_database=False)
@@ -83,18 +261,6 @@
'cube, which is not installed. Continue anyway?'):
raise
-if applcubicwebversion <= (3, 13, 0) and cubicwebversion >= (3, 13, 1):
- sql('ALTER TABLE entities ADD asource VARCHAR(64)')
- sql('UPDATE entities SET asource=cw_name '
- 'FROM cw_CWSource, cw_source_relation '
- 'WHERE entities.eid=cw_source_relation.eid_from AND cw_source_relation.eid_to=cw_CWSource.cw_eid')
- commit()
-
-if schema['TZDatetime'].eid is None:
- add_entity_type('TZDatetime', auto=False)
-if schema['TZTime'].eid is None:
- add_entity_type('TZTime', auto=False)
-
if applcubicwebversion <= (3, 14, 0) and cubicwebversion >= (3, 14, 0):
if 'require_permission' in schema and not 'localperms'in repo.config.cubes():
@@ -105,14 +271,13 @@
raise ExecutionError('In cubicweb 3.14, CWPermission and related stuff '
'has been moved to cube localperms. Install it first.')
+
if applcubicwebversion == (3, 6, 0) and cubicwebversion >= (3, 6, 0):
CSTRMAP = dict(rql('Any T, X WHERE X is CWConstraintType, X name T',
ask_confirm=False))
_add_relation_definition_no_perms('CWAttribute', 'update_permission', 'CWGroup')
_add_relation_definition_no_perms('CWAttribute', 'update_permission', 'RQLExpression')
rql('SET X update_permission Y WHERE X is CWAttribute, X add_permission Y')
- drop_relation_definition('CWAttribute', 'add_permission', 'CWGroup')
- drop_relation_definition('CWAttribute', 'add_permission', 'RQLExpression')
drop_relation_definition('CWAttribute', 'delete_permission', 'CWGroup')
drop_relation_definition('CWAttribute', 'delete_permission', 'RQLExpression')
--- a/server/sources/native.py Wed Mar 18 12:02:09 2015 +0100
+++ b/server/sources/native.py Tue Mar 17 15:50:40 2015 +0100
@@ -850,7 +850,7 @@
res = list(res)
if res[-1] is not None:
res[-1] = b64decode(res[-1])
- res.append(res[1])
+ res.append("system")
return res
def extid2eid(self, cnx, extid):