cubicweb/misc/migration/3.23.0_Any.py
author Philippe Pepiot <ph@itsalwaysdns.eu>
Tue, 31 Mar 2020 19:15:03 +0200
changeset 12957 0c973204033a
parent 12825 71a842bdf81d
permissions -rw-r--r--
[server] prevent returning closed cursor to the database pool In since c8c6ad8 init_repository use repo.internal_cnx() instead of repo.system_source.get_connection() so it use the pool and we should not close cursors from the pool before returning it back. Otherwise we may have "connection already closed" error. This bug only trigger when connection-pool-size = 1. Since we are moving to use a dynamic pooler we need to get this fixed. This does not occur with sqlite since the connection wrapper instantiate new cursor everytime, but this occur with other databases.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
11360
49aca289134f [sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11215
diff changeset
     1
49aca289134f [sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11215
diff changeset
     2
from functools import partial
49aca289134f [sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11215
diff changeset
     3
49aca289134f [sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11215
diff changeset
     4
from yams.constraints import UniqueConstraint
49aca289134f [sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11215
diff changeset
     5
49aca289134f [sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11215
diff changeset
     6
from cubicweb.schema import PURE_VIRTUAL_RTYPES
49aca289134f [sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11215
diff changeset
     7
from cubicweb.server.schema2sql import build_index_name, check_constraint
49aca289134f [sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11215
diff changeset
     8
49aca289134f [sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11215
diff changeset
     9
sql = partial(sql, ask_confirm=False)
49aca289134f [sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11215
diff changeset
    10
49aca289134f [sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11215
diff changeset
    11
source = repo.system_source
11363
e5fe836df6f1 [migration] Add IF EXISTS on DROP CONSTRAINT
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11360
diff changeset
    12
helper = source.dbhelper
11360
49aca289134f [sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11215
diff changeset
    13
11433
a703f00718c2 [migration] More resilient migration script
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11365
diff changeset
    14
# drop all relations primary keys
a703f00718c2 [migration] More resilient migration script
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11365
diff changeset
    15
for table, cstr in sql('''
a703f00718c2 [migration] More resilient migration script
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11365
diff changeset
    16
    SELECT DISTINCT tc.table_name, tc.constraint_name
a703f00718c2 [migration] More resilient migration script
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11365
diff changeset
    17
    FROM information_schema.table_constraints tc,
a703f00718c2 [migration] More resilient migration script
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11365
diff changeset
    18
         information_schema.key_column_usage kc
11845
eed83dee2c79 [migration] Fix 3.23.0 migration script, broken by a703f00718c2
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11433
diff changeset
    19
    WHERE tc.constraint_type = 'PRIMARY KEY'
11433
a703f00718c2 [migration] More resilient migration script
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11365
diff changeset
    20
          AND kc.table_name = tc.table_name
a703f00718c2 [migration] More resilient migration script
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11365
diff changeset
    21
          AND kc.table_name LIKE '%\_relation'
a703f00718c2 [migration] More resilient migration script
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11365
diff changeset
    22
          AND kc.table_schema = tc.table_schema
a703f00718c2 [migration] More resilient migration script
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11365
diff changeset
    23
          AND kc.constraint_name = tc.constraint_name;
a703f00718c2 [migration] More resilient migration script
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11365
diff changeset
    24
'''):
11845
eed83dee2c79 [migration] Fix 3.23.0 migration script, broken by a703f00718c2
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11433
diff changeset
    25
    sql('ALTER TABLE %s DROP CONSTRAINT %s' % (table, cstr))
11433
a703f00718c2 [migration] More resilient migration script
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11365
diff changeset
    26
a703f00718c2 [migration] More resilient migration script
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11365
diff changeset
    27
for table, cstr in sql("""
a703f00718c2 [migration] More resilient migration script
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11365
diff changeset
    28
    SELECT DISTINCT table_name, constraint_name FROM information_schema.constraint_column_usage
a703f00718c2 [migration] More resilient migration script
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11365
diff changeset
    29
    WHERE table_name LIKE 'cw\_%' AND constraint_name LIKE '%\_key'"""):
11845
eed83dee2c79 [migration] Fix 3.23.0 migration script, broken by a703f00718c2
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11433
diff changeset
    30
    sql('ALTER TABLE %s DROP CONSTRAINT %s' % (table, cstr))
11433
a703f00718c2 [migration] More resilient migration script
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11365
diff changeset
    31
11360
49aca289134f [sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11215
diff changeset
    32
for rschema in schema.relations():
49aca289134f [sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11215
diff changeset
    33
    if rschema.rule or rschema in PURE_VIRTUAL_RTYPES:
49aca289134f [sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11215
diff changeset
    34
        continue
49aca289134f [sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11215
diff changeset
    35
    if rschema.final or rschema.inlined:
49aca289134f [sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11215
diff changeset
    36
        for rdef in rschema.rdefs.values():
49aca289134f [sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11215
diff changeset
    37
            table = 'cw_{0}'.format(rdef.subject)
49aca289134f [sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11215
diff changeset
    38
            column = 'cw_{0}'.format(rdef.rtype)
49aca289134f [sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11215
diff changeset
    39
            if rschema.inlined or rdef.indexed:
49aca289134f [sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11215
diff changeset
    40
                old_name = '%s_%s_idx' % (table.lower(), column.lower())
49aca289134f [sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11215
diff changeset
    41
                sql('DROP INDEX IF EXISTS %s' % old_name)
49aca289134f [sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11215
diff changeset
    42
                source.create_index(cnx, table, column)
49aca289134f [sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11215
diff changeset
    43
    else:
49aca289134f [sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11215
diff changeset
    44
        table = '{0}_relation'.format(rschema)
49aca289134f [sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11215
diff changeset
    45
        sql('ALTER TABLE %s ADD CONSTRAINT %s PRIMARY KEY(eid_from, eid_to)'
49aca289134f [sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11215
diff changeset
    46
            % (table, build_index_name(table, ['eid_from', 'eid_to'], 'key_')))
49aca289134f [sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11215
diff changeset
    47
        for column in ('from', 'to'):
49aca289134f [sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11215
diff changeset
    48
            sql('DROP INDEX IF EXISTS %s_%s_idx' % (table, column))
49aca289134f [sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11215
diff changeset
    49
            sql('CREATE INDEX %s ON %s(eid_%s);'
49aca289134f [sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11215
diff changeset
    50
                % (build_index_name(table, ['eid_' + column], 'idx_'), table, column))
49aca289134f [sql gen] Control size of index name using an md5 hash
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11215
diff changeset
    51
11215
4e79acdc36a6 [schema] use json to serialize constraints
Julien Cristau <julien.cristau@logilab.fr>
parents:
diff changeset
    52
# we changed constraint serialization, which also changes their name
4e79acdc36a6 [schema] use json to serialize constraints
Julien Cristau <julien.cristau@logilab.fr>
parents:
diff changeset
    53
4e79acdc36a6 [schema] use json to serialize constraints
Julien Cristau <julien.cristau@logilab.fr>
parents:
diff changeset
    54
for table, cstr in sql("""
11433
a703f00718c2 [migration] More resilient migration script
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11365
diff changeset
    55
    SELECT DISTINCT table_name, constraint_name FROM information_schema.constraint_column_usage
11215
4e79acdc36a6 [schema] use json to serialize constraints
Julien Cristau <julien.cristau@logilab.fr>
parents:
diff changeset
    56
    WHERE constraint_name LIKE 'cstr%'"""):
11433
a703f00718c2 [migration] More resilient migration script
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11365
diff changeset
    57
    sql("ALTER TABLE %(table)s DROP CONSTRAINT %(cstr)s" % locals())
11215
4e79acdc36a6 [schema] use json to serialize constraints
Julien Cristau <julien.cristau@logilab.fr>
parents:
diff changeset
    58
4e79acdc36a6 [schema] use json to serialize constraints
Julien Cristau <julien.cristau@logilab.fr>
parents:
diff changeset
    59
for cwconstraint in rql('Any C WHERE R constrained_by C').entities():
4e79acdc36a6 [schema] use json to serialize constraints
Julien Cristau <julien.cristau@logilab.fr>
parents:
diff changeset
    60
    cwrdef = cwconstraint.reverse_constrained_by[0]
4e79acdc36a6 [schema] use json to serialize constraints
Julien Cristau <julien.cristau@logilab.fr>
parents:
diff changeset
    61
    rdef = cwrdef.yams_schema()
4e79acdc36a6 [schema] use json to serialize constraints
Julien Cristau <julien.cristau@logilab.fr>
parents:
diff changeset
    62
    cstr = rdef.constraint_by_eid(cwconstraint.eid)
11364
a702d31ddd8f [migration] Reserialize all constraints since yams introduce new JSON serializing format
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11363
diff changeset
    63
    with cnx.deny_all_hooks_but():
12825
71a842bdf81d [python3] replace 'unicode' by 'str'
Laurent Peuch <cortex@worlddomination.be>
parents: 11846
diff changeset
    64
        cwconstraint.cw_set(value=str(cstr.serialize()))
11215
4e79acdc36a6 [schema] use json to serialize constraints
Julien Cristau <julien.cristau@logilab.fr>
parents:
diff changeset
    65
    if cstr.type() not in ('BoundaryConstraint', 'IntervalBoundConstraint',
4e79acdc36a6 [schema] use json to serialize constraints
Julien Cristau <julien.cristau@logilab.fr>
parents:
diff changeset
    66
                           'StaticVocabularyConstraint'):
4e79acdc36a6 [schema] use json to serialize constraints
Julien Cristau <julien.cristau@logilab.fr>
parents:
diff changeset
    67
        # These cannot be translate into backend CHECK.
4e79acdc36a6 [schema] use json to serialize constraints
Julien Cristau <julien.cristau@logilab.fr>
parents:
diff changeset
    68
        continue
11406
8ed625765a5c [schema2sql] Give a rdef to check_constraint
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11365
diff changeset
    69
    cstrname, check = check_constraint(rdef, cstr, helper, prefix='cw_')
11215
4e79acdc36a6 [schema] use json to serialize constraints
Julien Cristau <julien.cristau@logilab.fr>
parents:
diff changeset
    70
    args = {'e': rdef.subject.type, 'c': cstrname, 'v': check}
4e79acdc36a6 [schema] use json to serialize constraints
Julien Cristau <julien.cristau@logilab.fr>
parents:
diff changeset
    71
    sql('ALTER TABLE cw_%(e)s ADD CONSTRAINT %(c)s CHECK(%(v)s)' % args)
4e79acdc36a6 [schema] use json to serialize constraints
Julien Cristau <julien.cristau@logilab.fr>
parents:
diff changeset
    72
4e79acdc36a6 [schema] use json to serialize constraints
Julien Cristau <julien.cristau@logilab.fr>
parents:
diff changeset
    73
commit()
11365
eedf0684533c [migration] drop identity_relation if it exists
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11364
diff changeset
    74
eedf0684533c [migration] drop identity_relation if it exists
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11364
diff changeset
    75
if 'identity_relation' in helper.list_tables(cnx.cnxset.cu):
eedf0684533c [migration] drop identity_relation if it exists
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents: 11364
diff changeset
    76
    sql('DROP TABLE identity_relation')