author | Julien Cristau <julien.cristau@logilab.fr> |
Fri, 17 Oct 2014 12:14:58 +0200 | |
changeset 10045 | 0a49d61c8cca |
parent 9797 | 4e640ab62f51 |
child 10365 | 21461f80f348 |
permissions | -rw-r--r-- |
9797
4e640ab62f51
[tests/checkintegrity] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9578
diff
changeset
|
1 |
# copyright 2003-2014 LOGILAB S.A. (Paris, FRANCE), all rights reserved. |
5421
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5341
diff
changeset
|
2 |
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5341
diff
changeset
|
3 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5341
diff
changeset
|
4 |
# This file is part of CubicWeb. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5341
diff
changeset
|
5 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5341
diff
changeset
|
6 |
# CubicWeb is free software: you can redistribute it and/or modify it under the |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5341
diff
changeset
|
7 |
# terms of the GNU Lesser General Public License as published by the Free |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5341
diff
changeset
|
8 |
# Software Foundation, either version 2.1 of the License, or (at your option) |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5341
diff
changeset
|
9 |
# any later version. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5341
diff
changeset
|
10 |
# |
5424
8ecbcbff9777
replace logilab-common by CubicWeb in disclaimer
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5421
diff
changeset
|
11 |
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT |
5421
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5341
diff
changeset
|
12 |
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5341
diff
changeset
|
13 |
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5341
diff
changeset
|
14 |
# details. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5341
diff
changeset
|
15 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5341
diff
changeset
|
16 |
# You should have received a copy of the GNU Lesser General Public License along |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5341
diff
changeset
|
17 |
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>. |
6127
747e423093fc
[ms, c-c] new command checking for consistency / potentian flaws and enhancements of mapping file of a multi-sources instance
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6112
diff
changeset
|
18 |
"""Integrity checking tool for instances: |
747e423093fc
[ms, c-c] new command checking for consistency / potentian flaws and enhancements of mapping file of a multi-sources instance
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6112
diff
changeset
|
19 |
|
747e423093fc
[ms, c-c] new command checking for consistency / potentian flaws and enhancements of mapping file of a multi-sources instance
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6112
diff
changeset
|
20 |
* integrity of a CubicWeb repository. Hum actually only the system database is |
747e423093fc
[ms, c-c] new command checking for consistency / potentian flaws and enhancements of mapping file of a multi-sources instance
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6112
diff
changeset
|
21 |
checked. |
5999
eaf8219f8b7d
[migration] fix rename_entity_type to avoid to loose some relations on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5954
diff
changeset
|
22 |
""" |
0 | 23 |
__docformat__ = "restructuredtext en" |
24 |
||
25 |
import sys |
|
1016
26387b836099
use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents:
713
diff
changeset
|
26 |
from datetime import datetime |
0 | 27 |
|
28 |
from logilab.common.shellutils import ProgressBar |
|
29 |
||
7468
c202aaf71489
[server] fix NameError (missing VIRTUAL_RTYPES import) in checkintegrity.py
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
7040
diff
changeset
|
30 |
from cubicweb.schema import PURE_VIRTUAL_RTYPES, VIRTUAL_RTYPES |
1251
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1161
diff
changeset
|
31 |
from cubicweb.server.sqlutils import SQL_PREFIX |
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1161
diff
changeset
|
32 |
|
7035
8d2cf36bd79d
[c-c db-check] factorize code by introducing notify_fixed dumb function
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6889
diff
changeset
|
33 |
def notify_fixed(fix): |
8d2cf36bd79d
[c-c db-check] factorize code by introducing notify_fixed dumb function
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6889
diff
changeset
|
34 |
if fix: |
7896
4c954e1e73ef
[lint] remove uses of "print >> sys.stderr" (closes #1908571)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7815
diff
changeset
|
35 |
sys.stderr.write(' [FIXED]') |
4c954e1e73ef
[lint] remove uses of "print >> sys.stderr" (closes #1908571)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7815
diff
changeset
|
36 |
sys.stderr.write('\n') |
7035
8d2cf36bd79d
[c-c db-check] factorize code by introducing notify_fixed dumb function
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6889
diff
changeset
|
37 |
|
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
38 |
def has_eid(cnx, sqlcursor, eid, eids): |
0 | 39 |
"""return true if the eid is a valid eid""" |
5341
0de53140bd29
[db-check] cleanup
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5340
diff
changeset
|
40 |
if eid in eids: |
0 | 41 |
return eids[eid] |
9469
032825bbacab
[multi-sources-removal] Drop entities.source column
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9468
diff
changeset
|
42 |
sqlcursor.execute('SELECT type FROM entities WHERE eid=%s' % eid) |
0 | 43 |
try: |
9469
032825bbacab
[multi-sources-removal] Drop entities.source column
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9468
diff
changeset
|
44 |
etype = sqlcursor.fetchone()[0] |
7815
2a164a9cf81c
[exceptions] stop catching any exception in various places (closes #1942716)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7687
diff
changeset
|
45 |
except Exception: |
0 | 46 |
eids[eid] = False |
47 |
return False |
|
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
48 |
if etype not in cnx.vreg.schema: |
9171
be9596750678
Fix two crashes in db-check (closes #3024964)
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
8900
diff
changeset
|
49 |
eids[eid] = False |
be9596750678
Fix two crashes in db-check (closes #3024964)
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
8900
diff
changeset
|
50 |
return False |
1251
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1161
diff
changeset
|
51 |
sqlcursor.execute('SELECT * FROM %s%s WHERE %seid=%s' % (SQL_PREFIX, etype, |
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1161
diff
changeset
|
52 |
SQL_PREFIX, eid)) |
0 | 53 |
result = sqlcursor.fetchall() |
54 |
if len(result) == 0: |
|
55 |
eids[eid] = False |
|
56 |
return False |
|
57 |
elif len(result) > 1: |
|
9492
c7fc56eecd1a
English typography
Dimitri Papadopoulos <dimitri.papadopoulos@cea.fr>
parents:
9171
diff
changeset
|
58 |
msg = (' More than one entity with eid %s exists in source!\n' |
c7fc56eecd1a
English typography
Dimitri Papadopoulos <dimitri.papadopoulos@cea.fr>
parents:
9171
diff
changeset
|
59 |
' WARNING : Unable to fix this, do it yourself!\n') |
7896
4c954e1e73ef
[lint] remove uses of "print >> sys.stderr" (closes #1908571)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7815
diff
changeset
|
60 |
sys.stderr.write(msg % eid) |
0 | 61 |
eids[eid] = True |
62 |
return True |
|
63 |
||
64 |
# XXX move to yams? |
|
65 |
def etype_fti_containers(eschema, _done=None): |
|
66 |
if _done is None: |
|
67 |
_done = set() |
|
68 |
_done.add(eschema) |
|
69 |
containers = tuple(eschema.fulltext_containers()) |
|
70 |
if containers: |
|
71 |
for rschema, target in containers: |
|
72 |
if target == 'object': |
|
73 |
targets = rschema.objects(eschema) |
|
74 |
else: |
|
75 |
targets = rschema.subjects(eschema) |
|
76 |
for targeteschema in targets: |
|
77 |
if targeteschema in _done: |
|
78 |
continue |
|
79 |
_done.add(targeteschema) |
|
80 |
for container in etype_fti_containers(targeteschema, _done): |
|
81 |
yield container |
|
82 |
else: |
|
83 |
yield eschema |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
84 |
|
9574
2d4c4842bd04
[server] some s/session/cnx/
Julien Cristau <julien.cristau@logilab.fr>
parents:
9543
diff
changeset
|
85 |
def reindex_entities(schema, cnx, withpb=True, etypes=None): |
0 | 86 |
"""reindex all entities in the repository""" |
87 |
# deactivate modification_date hook since we don't want them |
|
88 |
# to be updated due to the reindexation |
|
9574
2d4c4842bd04
[server] some s/session/cnx/
Julien Cristau <julien.cristau@logilab.fr>
parents:
9543
diff
changeset
|
89 |
repo = cnx.repo |
2d4c4842bd04
[server] some s/session/cnx/
Julien Cristau <julien.cristau@logilab.fr>
parents:
9543
diff
changeset
|
90 |
dbhelper = repo.system_source.dbhelper |
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
91 |
with cnx.ensure_cnx_set: |
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
92 |
cursor = cnx.cnxset.cu |
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
93 |
if not dbhelper.has_fti_table(cursor): |
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
94 |
print 'no text index table' |
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
95 |
dbhelper.init_fti(cursor) |
4806
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4691
diff
changeset
|
96 |
repo.system_source.do_fti = True # ensure full-text indexation is activated |
5850
fabff2813ee4
[migration] schema should be accessed through .repo
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5693
diff
changeset
|
97 |
if etypes is None: |
5954
987086484876
[fti migration] test and fix reindexation of some specific entity types
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5850
diff
changeset
|
98 |
print 'Reindexing entities' |
5850
fabff2813ee4
[migration] schema should be accessed through .repo
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5693
diff
changeset
|
99 |
etypes = set() |
fabff2813ee4
[migration] schema should be accessed through .repo
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5693
diff
changeset
|
100 |
for eschema in schema.entities(): |
fabff2813ee4
[migration] schema should be accessed through .repo
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5693
diff
changeset
|
101 |
if eschema.final: |
fabff2813ee4
[migration] schema should be accessed through .repo
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5693
diff
changeset
|
102 |
continue |
fabff2813ee4
[migration] schema should be accessed through .repo
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5693
diff
changeset
|
103 |
indexable_attrs = tuple(eschema.indexable_attributes()) # generator |
fabff2813ee4
[migration] schema should be accessed through .repo
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5693
diff
changeset
|
104 |
if not indexable_attrs: |
fabff2813ee4
[migration] schema should be accessed through .repo
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5693
diff
changeset
|
105 |
continue |
fabff2813ee4
[migration] schema should be accessed through .repo
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5693
diff
changeset
|
106 |
for container in etype_fti_containers(eschema): |
fabff2813ee4
[migration] schema should be accessed through .repo
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5693
diff
changeset
|
107 |
etypes.add(container) |
5954
987086484876
[fti migration] test and fix reindexation of some specific entity types
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5850
diff
changeset
|
108 |
# clear fti table first |
9574
2d4c4842bd04
[server] some s/session/cnx/
Julien Cristau <julien.cristau@logilab.fr>
parents:
9543
diff
changeset
|
109 |
cnx.system_sql('DELETE FROM %s' % dbhelper.fti_table) |
5954
987086484876
[fti migration] test and fix reindexation of some specific entity types
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5850
diff
changeset
|
110 |
else: |
987086484876
[fti migration] test and fix reindexation of some specific entity types
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5850
diff
changeset
|
111 |
print 'Reindexing entities of type %s' % \ |
987086484876
[fti migration] test and fix reindexation of some specific entity types
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5850
diff
changeset
|
112 |
', '.join(sorted(str(e) for e in etypes)) |
987086484876
[fti migration] test and fix reindexation of some specific entity types
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5850
diff
changeset
|
113 |
# clear fti table first. Use subquery for sql compatibility |
9574
2d4c4842bd04
[server] some s/session/cnx/
Julien Cristau <julien.cristau@logilab.fr>
parents:
9543
diff
changeset
|
114 |
cnx.system_sql("DELETE FROM %s WHERE EXISTS(SELECT 1 FROM ENTITIES " |
2d4c4842bd04
[server] some s/session/cnx/
Julien Cristau <julien.cristau@logilab.fr>
parents:
9543
diff
changeset
|
115 |
"WHERE eid=%s AND type IN (%s))" % ( |
2d4c4842bd04
[server] some s/session/cnx/
Julien Cristau <julien.cristau@logilab.fr>
parents:
9543
diff
changeset
|
116 |
dbhelper.fti_table, dbhelper.fti_uid_attr, |
2d4c4842bd04
[server] some s/session/cnx/
Julien Cristau <julien.cristau@logilab.fr>
parents:
9543
diff
changeset
|
117 |
','.join("'%s'" % etype for etype in etypes))) |
4675
9233a8350420
[test] don't display progress bar when testing checkintegrity
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
118 |
if withpb: |
6112
913979c79244
[db-fti-index] simple fix fpr progressbar-related crash when etypes is None in reindex_entities()
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
5999
diff
changeset
|
119 |
pb = ProgressBar(len(etypes) + 1) |
4675
9233a8350420
[test] don't display progress bar when testing checkintegrity
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
120 |
pb.update() |
0 | 121 |
# reindex entities by generating rql queries which set all indexable |
122 |
# attribute to their current value |
|
4816
c02583cb80a9
repair stuff broken by fti handling changes
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4806
diff
changeset
|
123 |
source = repo.system_source |
0 | 124 |
for eschema in etypes: |
9574
2d4c4842bd04
[server] some s/session/cnx/
Julien Cristau <julien.cristau@logilab.fr>
parents:
9543
diff
changeset
|
125 |
etype_class = cnx.vreg['etypes'].etype_class(str(eschema)) |
2d4c4842bd04
[server] some s/session/cnx/
Julien Cristau <julien.cristau@logilab.fr>
parents:
9543
diff
changeset
|
126 |
for fti_rql in etype_class.cw_fti_index_rql_queries(cnx): |
2d4c4842bd04
[server] some s/session/cnx/
Julien Cristau <julien.cristau@logilab.fr>
parents:
9543
diff
changeset
|
127 |
rset = cnx.execute(fti_rql) |
2d4c4842bd04
[server] some s/session/cnx/
Julien Cristau <julien.cristau@logilab.fr>
parents:
9543
diff
changeset
|
128 |
source.fti_index_entities(cnx, rset.entities()) |
8450
11063635c4e4
[fti] allow usage of custom RQL to fetch entities to index (closes #2410509)
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
8447
diff
changeset
|
129 |
# clear entity cache to avoid high memory consumption on big tables |
9574
2d4c4842bd04
[server] some s/session/cnx/
Julien Cristau <julien.cristau@logilab.fr>
parents:
9543
diff
changeset
|
130 |
cnx.drop_entity_cache() |
4675
9233a8350420
[test] don't display progress bar when testing checkintegrity
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
131 |
if withpb: |
9233a8350420
[test] don't display progress bar when testing checkintegrity
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
132 |
pb.update() |
9578
68049d53426c
[fti] properly close the ProgressBar
Julien Cristau <julien.cristau@logilab.fr>
parents:
9577
diff
changeset
|
133 |
if withpb: |
68049d53426c
[fti] properly close the ProgressBar
Julien Cristau <julien.cristau@logilab.fr>
parents:
9577
diff
changeset
|
134 |
pb.finish() |
0 | 135 |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
136 |
|
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
137 |
def check_schema(schema, cnx, eids, fix=1): |
0 | 138 |
"""check serialized schema""" |
139 |
print 'Checking serialized schema' |
|
140 |
unique_constraints = ('SizeConstraint', 'FormatConstraint', |
|
5523
4bf975c049a6
[db-check] RQLConstraint is not a 'unique' constraint
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
141 |
'VocabularyConstraint', |
0 | 142 |
'RQLVocabularyConstraint') |
5338
3e5a256d17ba
[db-check] fix duplicated schema constraint detection
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4835
diff
changeset
|
143 |
rql = ('Any COUNT(X),RN,SN,ON,CTN GROUPBY RN,SN,ON,CTN ORDERBY 1 ' |
1398
5fe84a5f7035
rename internal entity types to have CW prefix instead of E
sylvain.thenault@logilab.fr
parents:
1263
diff
changeset
|
144 |
'WHERE X is CWConstraint, R constrained_by X, ' |
5338
3e5a256d17ba
[db-check] fix duplicated schema constraint detection
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4835
diff
changeset
|
145 |
'R relation_type RT, RT name RN, R from_entity ST, ST name SN, ' |
3e5a256d17ba
[db-check] fix duplicated schema constraint detection
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4835
diff
changeset
|
146 |
'R to_entity OT, OT name ON, X cstrtype CT, CT name CTN') |
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
147 |
for count, rn, sn, on, cstrname in cnx.execute(rql): |
0 | 148 |
if count == 1: |
149 |
continue |
|
150 |
if cstrname in unique_constraints: |
|
5338
3e5a256d17ba
[db-check] fix duplicated schema constraint detection
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4835
diff
changeset
|
151 |
print "ERROR: got %s %r constraints on relation %s.%s.%s" % ( |
3e5a256d17ba
[db-check] fix duplicated schema constraint detection
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4835
diff
changeset
|
152 |
count, cstrname, sn, rn, on) |
5523
4bf975c049a6
[db-check] RQLConstraint is not a 'unique' constraint
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
153 |
if fix: |
4bf975c049a6
[db-check] RQLConstraint is not a 'unique' constraint
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5424
diff
changeset
|
154 |
print 'dunno how to fix, do it yourself' |
0 | 155 |
|
156 |
||
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
157 |
|
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
158 |
def check_text_index(schema, cnx, eids, fix=1): |
0 | 159 |
"""check all entities registered in the text index""" |
160 |
print 'Checking text index' |
|
7965
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
161 |
msg = ' Entity with eid %s exists in the text index but in no source (autofix will remove from text index)' |
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
162 |
cursor = cnx.system_sql('SELECT uid FROM appears;') |
0 | 163 |
for row in cursor.fetchall(): |
164 |
eid = row[0] |
|
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
165 |
if not has_eid(cnx, cursor, eid, eids): |
7896
4c954e1e73ef
[lint] remove uses of "print >> sys.stderr" (closes #1908571)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7815
diff
changeset
|
166 |
sys.stderr.write(msg % eid) |
0 | 167 |
if fix: |
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
168 |
cnx.system_sql('DELETE FROM appears WHERE uid=%s;' % eid) |
7035
8d2cf36bd79d
[c-c db-check] factorize code by introducing notify_fixed dumb function
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6889
diff
changeset
|
169 |
notify_fixed(fix) |
0 | 170 |
|
171 |
||
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
172 |
def check_entities(schema, cnx, eids, fix=1): |
0 | 173 |
"""check all entities registered in the repo system table""" |
174 |
print 'Checking entities system table' |
|
7965
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
175 |
# system table but no source |
9171
be9596750678
Fix two crashes in db-check (closes #3024964)
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
8900
diff
changeset
|
176 |
msg = ' Entity %s with eid %s exists in the system table but in no source (autofix will delete the entity)' |
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
177 |
cursor = cnx.system_sql('SELECT eid,type FROM entities;') |
0 | 178 |
for row in cursor.fetchall(): |
9171
be9596750678
Fix two crashes in db-check (closes #3024964)
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
8900
diff
changeset
|
179 |
eid, etype = row |
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
180 |
if not has_eid(cnx, cursor, eid, eids): |
9171
be9596750678
Fix two crashes in db-check (closes #3024964)
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
8900
diff
changeset
|
181 |
sys.stderr.write(msg % (etype, eid)) |
0 | 182 |
if fix: |
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
183 |
cnx.system_sql('DELETE FROM entities WHERE eid=%s;' % eid) |
7035
8d2cf36bd79d
[c-c db-check] factorize code by introducing notify_fixed dumb function
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6889
diff
changeset
|
184 |
notify_fixed(fix) |
7965
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
185 |
# source in entities, but no relation cw_source |
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
186 |
# XXX this (get_versions) requires a second connection to the db when we already have one open |
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
187 |
applcwversion = cnx.repo.get_versions().get('cubicweb') |
9468
39b7a91a3f4c
[repo] pylint cleanup, mainly of imports, with a bit of style
Julien Cristau <julien.cristau@logilab.fr>
parents:
9463
diff
changeset
|
188 |
if applcwversion >= (3, 13, 1): # entities.asource appeared in 3.13.1 |
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
189 |
cursor = cnx.system_sql('SELECT e.eid FROM entities as e, cw_CWSource as s ' |
7965
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
190 |
'WHERE s.cw_name=e.asource AND ' |
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
191 |
'NOT EXISTS(SELECT 1 FROM cw_source_relation as cs ' |
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
192 |
' WHERE cs.eid_from=e.eid AND cs.eid_to=s.cw_eid) ' |
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
193 |
'ORDER BY e.eid') |
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
194 |
msg = (' Entity with eid %s refers to source in entities table, ' |
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
195 |
'but is missing relation cw_source (autofix will create the relation)\n') |
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
196 |
for row in cursor.fetchall(): |
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
197 |
sys.stderr.write(msg % row[0]) |
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
198 |
if fix: |
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
199 |
cnx.system_sql('INSERT INTO cw_source_relation (eid_from, eid_to) ' |
7965
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
200 |
'SELECT e.eid, s.cw_eid FROM entities as e, cw_CWSource as s ' |
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
201 |
'WHERE s.cw_name=e.asource AND NOT EXISTS(SELECT 1 FROM cw_source_relation as cs ' |
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
202 |
' WHERE cs.eid_from=e.eid AND cs.eid_to=s.cw_eid)') |
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
203 |
notify_fixed(True) |
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
204 |
# inconsistencies for 'is' |
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
205 |
msg = ' %s #%s is missing relation "is" (autofix will create the relation)\n' |
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
206 |
cursor = cnx.system_sql('SELECT e.type, e.eid FROM entities as e, cw_CWEType as s ' |
7965
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
207 |
'WHERE s.cw_name=e.type AND NOT EXISTS(SELECT 1 FROM is_relation as cs ' |
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
208 |
' WHERE cs.eid_from=e.eid AND cs.eid_to=s.cw_eid) ' |
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
209 |
'ORDER BY e.eid') |
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
210 |
for row in cursor.fetchall(): |
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
211 |
sys.stderr.write(msg % row) |
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
212 |
if fix: |
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
213 |
cnx.system_sql('INSERT INTO is_relation (eid_from, eid_to) ' |
7965
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
214 |
'SELECT e.eid, s.cw_eid FROM entities as e, cw_CWEType as s ' |
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
215 |
'WHERE s.cw_name=e.type AND NOT EXISTS(SELECT 1 FROM is_relation as cs ' |
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
216 |
' WHERE cs.eid_from=e.eid AND cs.eid_to=s.cw_eid)') |
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
217 |
notify_fixed(True) |
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
218 |
# inconsistencies for 'is_instance_of' |
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
219 |
msg = ' %s #%s is missing relation "is_instance_of" (autofix will create the relation)\n' |
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
220 |
cursor = cnx.system_sql('SELECT e.type, e.eid FROM entities as e, cw_CWEType as s ' |
7965
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
221 |
'WHERE s.cw_name=e.type AND NOT EXISTS(SELECT 1 FROM is_instance_of_relation as cs ' |
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
222 |
' WHERE cs.eid_from=e.eid AND cs.eid_to=s.cw_eid) ' |
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
223 |
'ORDER BY e.eid') |
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
224 |
for row in cursor.fetchall(): |
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
225 |
sys.stderr.write(msg % row) |
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
226 |
if fix: |
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
227 |
cnx.system_sql('INSERT INTO is_instance_of_relation (eid_from, eid_to) ' |
7965
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
228 |
'SELECT e.eid, s.cw_eid FROM entities as e, cw_CWEType as s ' |
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
229 |
'WHERE s.cw_name=e.type AND NOT EXISTS(SELECT 1 FROM is_instance_of_relation as cs ' |
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
230 |
' WHERE cs.eid_from=e.eid AND cs.eid_to=s.cw_eid)') |
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
231 |
notify_fixed(True) |
0 | 232 |
print 'Checking entities tables' |
7965
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
233 |
msg = ' Entity with eid %s exists in the %s table but not in the system table (autofix will delete the entity)' |
0 | 234 |
for eschema in schema.entities(): |
3689
deb13e88e037
follow yams 0.25 api changes to improve performance
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3374
diff
changeset
|
235 |
if eschema.final: |
0 | 236 |
continue |
1251
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1161
diff
changeset
|
237 |
table = SQL_PREFIX + eschema.type |
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1161
diff
changeset
|
238 |
column = SQL_PREFIX + 'eid' |
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
239 |
cursor = cnx.system_sql('SELECT %s FROM %s;' % (column, table)) |
0 | 240 |
for row in cursor.fetchall(): |
241 |
eid = row[0] |
|
5341
0de53140bd29
[db-check] cleanup
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5340
diff
changeset
|
242 |
# eids is full since we have fetched everything from the entities table, |
0 | 243 |
# no need to call has_eid |
244 |
if not eid in eids or not eids[eid]: |
|
7896
4c954e1e73ef
[lint] remove uses of "print >> sys.stderr" (closes #1908571)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7815
diff
changeset
|
245 |
sys.stderr.write(msg % (eid, eschema.type)) |
0 | 246 |
if fix: |
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
247 |
cnx.system_sql('DELETE FROM %s WHERE %s=%s;' % (table, column, eid)) |
7035
8d2cf36bd79d
[c-c db-check] factorize code by introducing notify_fixed dumb function
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6889
diff
changeset
|
248 |
notify_fixed(fix) |
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
249 |
|
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
250 |
|
0 | 251 |
def bad_related_msg(rtype, target, eid, fix): |
252 |
msg = ' A relation %s with %s eid %s exists but no such entity in sources' |
|
7896
4c954e1e73ef
[lint] remove uses of "print >> sys.stderr" (closes #1908571)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7815
diff
changeset
|
253 |
sys.stderr.write(msg % (rtype, target, eid)) |
7035
8d2cf36bd79d
[c-c db-check] factorize code by introducing notify_fixed dumb function
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6889
diff
changeset
|
254 |
notify_fixed(fix) |
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
255 |
|
9171
be9596750678
Fix two crashes in db-check (closes #3024964)
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
8900
diff
changeset
|
256 |
def bad_inlined_msg(rtype, parent_eid, eid, fix): |
be9596750678
Fix two crashes in db-check (closes #3024964)
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
8900
diff
changeset
|
257 |
msg = (' An inlined relation %s from %s to %s exists but the latter ' |
be9596750678
Fix two crashes in db-check (closes #3024964)
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
8900
diff
changeset
|
258 |
'entity does not exist') |
be9596750678
Fix two crashes in db-check (closes #3024964)
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
8900
diff
changeset
|
259 |
sys.stderr.write(msg % (rtype, parent_eid, eid)) |
be9596750678
Fix two crashes in db-check (closes #3024964)
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
8900
diff
changeset
|
260 |
notify_fixed(fix) |
be9596750678
Fix two crashes in db-check (closes #3024964)
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
8900
diff
changeset
|
261 |
|
1802
d628defebc17
delete-trailing-whitespace + some copyright update
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
1398
diff
changeset
|
262 |
|
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
263 |
def check_relations(schema, cnx, eids, fix=1): |
7035
8d2cf36bd79d
[c-c db-check] factorize code by introducing notify_fixed dumb function
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6889
diff
changeset
|
264 |
"""check that eids referenced by relations are registered in the repo system |
8d2cf36bd79d
[c-c db-check] factorize code by introducing notify_fixed dumb function
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6889
diff
changeset
|
265 |
table |
8d2cf36bd79d
[c-c db-check] factorize code by introducing notify_fixed dumb function
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6889
diff
changeset
|
266 |
""" |
0 | 267 |
print 'Checking relations' |
268 |
for rschema in schema.relations(): |
|
7965
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
269 |
if rschema.final or rschema.type in PURE_VIRTUAL_RTYPES: |
0 | 270 |
continue |
271 |
if rschema.inlined: |
|
272 |
for subjtype in rschema.subjects(): |
|
1251
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1161
diff
changeset
|
273 |
table = SQL_PREFIX + str(subjtype) |
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1161
diff
changeset
|
274 |
column = SQL_PREFIX + str(rschema) |
9171
be9596750678
Fix two crashes in db-check (closes #3024964)
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
8900
diff
changeset
|
275 |
sql = 'SELECT cw_eid,%s FROM %s WHERE %s IS NOT NULL;' % ( |
1251
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1161
diff
changeset
|
276 |
column, table, column) |
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
277 |
cursor = cnx.system_sql(sql) |
0 | 278 |
for row in cursor.fetchall(): |
9171
be9596750678
Fix two crashes in db-check (closes #3024964)
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
8900
diff
changeset
|
279 |
parent_eid, eid = row |
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
280 |
if not has_eid(cnx, cursor, eid, eids): |
9171
be9596750678
Fix two crashes in db-check (closes #3024964)
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
8900
diff
changeset
|
281 |
bad_inlined_msg(rschema, parent_eid, eid, fix) |
0 | 282 |
if fix: |
3374
d5bd1b659ce8
[db-check] fix sql to fix bad eid referenced by inlined relation
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2596
diff
changeset
|
283 |
sql = 'UPDATE %s SET %s=NULL WHERE %s=%s;' % ( |
d5bd1b659ce8
[db-check] fix sql to fix bad eid referenced by inlined relation
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2596
diff
changeset
|
284 |
table, column, column, eid) |
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
285 |
cnx.system_sql(sql) |
0 | 286 |
continue |
6185
229006accd26
[c-c db-check] skip error while checking relation, useful when analyzing really broken database (after a migration failure for instance
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6132
diff
changeset
|
287 |
try: |
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
288 |
cursor = cnx.system_sql('SELECT eid_from FROM %s_relation;' % rschema) |
8695
358d8bed9626
[toward-py3k] rewrite to "except AnException as exc:" (part of #2711624)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
8694
diff
changeset
|
289 |
except Exception as ex: |
6185
229006accd26
[c-c db-check] skip error while checking relation, useful when analyzing really broken database (after a migration failure for instance
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6132
diff
changeset
|
290 |
# usually because table doesn't exist |
229006accd26
[c-c db-check] skip error while checking relation, useful when analyzing really broken database (after a migration failure for instance
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6132
diff
changeset
|
291 |
print 'ERROR', ex |
229006accd26
[c-c db-check] skip error while checking relation, useful when analyzing really broken database (after a migration failure for instance
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6132
diff
changeset
|
292 |
continue |
0 | 293 |
for row in cursor.fetchall(): |
294 |
eid = row[0] |
|
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
295 |
if not has_eid(cnx, cursor, eid, eids): |
1251
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1161
diff
changeset
|
296 |
bad_related_msg(rschema, 'subject', eid, fix) |
0 | 297 |
if fix: |
380 | 298 |
sql = 'DELETE FROM %s_relation WHERE eid_from=%s;' % ( |
1251
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1161
diff
changeset
|
299 |
rschema, eid) |
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
300 |
cnx.system_sql(sql) |
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
301 |
cursor = cnx.system_sql('SELECT eid_to FROM %s_relation;' % rschema) |
0 | 302 |
for row in cursor.fetchall(): |
303 |
eid = row[0] |
|
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
304 |
if not has_eid(cnx, cursor, eid, eids): |
1251
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1161
diff
changeset
|
305 |
bad_related_msg(rschema, 'object', eid, fix) |
0 | 306 |
if fix: |
380 | 307 |
sql = 'DELETE FROM %s_relation WHERE eid_to=%s;' % ( |
1251
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1161
diff
changeset
|
308 |
rschema, eid) |
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
309 |
cnx.system_sql(sql) |
0 | 310 |
|
311 |
||
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
312 |
def check_mandatory_relations(schema, cnx, eids, fix=1): |
7036
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
313 |
"""check entities missing some mandatory relation""" |
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
314 |
print 'Checking mandatory relations' |
7965
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
315 |
msg = '%s #%s is missing mandatory %s relation %s (autofix will delete the entity)' |
7036
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
316 |
for rschema in schema.relations(): |
8527
6151b0199bc1
[c-c db-check] don't skip is/is_instance_of mandatory relation to avoid telling two opposite messages when the relation is missing. Closes #2465659
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8450
diff
changeset
|
317 |
if rschema.final or rschema in PURE_VIRTUAL_RTYPES or rschema in ('is', 'is_instance_of'): |
7036
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
318 |
continue |
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
319 |
smandatory = set() |
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
320 |
omandatory = set() |
8696
0bb18407c053
[toward py3k] rewrite dict.keys() and dict.values() (part of #2711624)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
8695
diff
changeset
|
321 |
for rdef in rschema.rdefs.itervalues(): |
7036
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
322 |
if rdef.cardinality[0] in '1+': |
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
323 |
smandatory.add(rdef.subject) |
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
324 |
if rdef.cardinality[1] in '1+': |
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
325 |
omandatory.add(rdef.object) |
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
326 |
for role, etypes in (('subject', smandatory), ('object', omandatory)): |
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
327 |
for etype in etypes: |
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
328 |
if role == 'subject': |
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
329 |
rql = 'Any X WHERE NOT X %s Y, X is %s' % (rschema, etype) |
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
330 |
else: |
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
331 |
rql = 'Any X WHERE NOT Y %s X, X is %s' % (rschema, etype) |
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
332 |
for entity in cnx.execute(rql).entities(): |
8900
010a59e12d89
use cw_etype instead of __regid__
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
8696
diff
changeset
|
333 |
sys.stderr.write(msg % (entity.cw_etype, entity.eid, role, rschema)) |
7036
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
334 |
if fix: |
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
335 |
#if entity.cw_describe()['source']['uri'] == 'system': XXX |
7965
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
336 |
entity.cw_delete() # XXX this is BRUTAL! |
7036
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
337 |
notify_fixed(fix) |
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
338 |
|
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
339 |
|
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
340 |
def check_mandatory_attributes(schema, cnx, eids, fix=1): |
7036
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
341 |
"""check for entities stored in the system source missing some mandatory |
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
342 |
attribute |
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
343 |
""" |
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
344 |
print 'Checking mandatory attributes' |
7965
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
345 |
msg = '%s #%s is missing mandatory attribute %s (autofix will delete the entity)' |
7036
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
346 |
for rschema in schema.relations(): |
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
347 |
if not rschema.final or rschema in VIRTUAL_RTYPES: |
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
348 |
continue |
8696
0bb18407c053
[toward py3k] rewrite dict.keys() and dict.values() (part of #2711624)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
8695
diff
changeset
|
349 |
for rdef in rschema.rdefs.itervalues(): |
7036
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
350 |
if rdef.cardinality[0] in '1+': |
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
351 |
rql = 'Any X WHERE X %s NULL, X is %s, X cw_source S, S name "system"' % ( |
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
352 |
rschema, rdef.subject) |
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
353 |
for entity in cnx.execute(rql).entities(): |
8900
010a59e12d89
use cw_etype instead of __regid__
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
8696
diff
changeset
|
354 |
sys.stderr.write(msg % (entity.cw_etype, entity.eid, rschema)) |
7036
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
355 |
if fix: |
7682
07b592a62be3
[c-c db-check] drop some deprecation warning
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7473
diff
changeset
|
356 |
entity.cw_delete() |
7036
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
357 |
notify_fixed(fix) |
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
358 |
|
63386b35ec69
[c-c db-check] new checks for entities missing a mandatory relation/attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7035
diff
changeset
|
359 |
|
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
360 |
def check_metadata(schema, cnx, eids, fix=1): |
0 | 361 |
"""check entities has required metadata |
362 |
||
363 |
FIXME: rewrite using RQL queries ? |
|
364 |
""" |
|
365 |
print 'Checking metadata' |
|
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
366 |
cursor = cnx.system_sql("SELECT DISTINCT type FROM entities;") |
1251
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1161
diff
changeset
|
367 |
eidcolumn = SQL_PREFIX + 'eid' |
7965
d45c0eb39e72
[server] fix integrity checks that destroy data (closes #1972842)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7896
diff
changeset
|
368 |
msg = ' %s with eid %s has no %s (autofix will set it to now)' |
0 | 369 |
for etype, in cursor.fetchall(): |
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
370 |
if etype not in cnx.vreg.schema: |
9171
be9596750678
Fix two crashes in db-check (closes #3024964)
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
8900
diff
changeset
|
371 |
sys.stderr.write('entities table references unknown type %s\n' % |
be9596750678
Fix two crashes in db-check (closes #3024964)
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
8900
diff
changeset
|
372 |
etype) |
be9596750678
Fix two crashes in db-check (closes #3024964)
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
8900
diff
changeset
|
373 |
if fix: |
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
374 |
cnx.system_sql("DELETE FROM entities WHERE type = %(type)s", |
9171
be9596750678
Fix two crashes in db-check (closes #3024964)
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
8900
diff
changeset
|
375 |
{'type': etype}) |
be9596750678
Fix two crashes in db-check (closes #3024964)
Denis Laxalde <denis.laxalde@logilab.fr>
parents:
8900
diff
changeset
|
376 |
continue |
1251
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1161
diff
changeset
|
377 |
table = SQL_PREFIX + etype |
1016
26387b836099
use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents:
713
diff
changeset
|
378 |
for rel, default in ( ('creation_date', datetime.now()), |
26387b836099
use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents:
713
diff
changeset
|
379 |
('modification_date', datetime.now()), ): |
1251
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1161
diff
changeset
|
380 |
column = SQL_PREFIX + rel |
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
381 |
cursor = cnx.system_sql("SELECT %s FROM %s WHERE %s is NULL" |
1251
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1161
diff
changeset
|
382 |
% (eidcolumn, table, column)) |
0 | 383 |
for eid, in cursor.fetchall(): |
7896
4c954e1e73ef
[lint] remove uses of "print >> sys.stderr" (closes #1908571)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
7815
diff
changeset
|
384 |
sys.stderr.write(msg % (etype, eid, rel)) |
0 | 385 |
if fix: |
9577
c52441e4a3d7
[serverctl] use repoapi for db-check, add-source, rebuild-fti commands
Julien Cristau <julien.cristau@logilab.fr>
parents:
9574
diff
changeset
|
386 |
cnx.system_sql("UPDATE %s SET %s=%%(v)s WHERE %s=%s ;" |
1251
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1161
diff
changeset
|
387 |
% (table, column, eidcolumn, eid), |
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1161
diff
changeset
|
388 |
{'v': default}) |
7035
8d2cf36bd79d
[c-c db-check] factorize code by introducing notify_fixed dumb function
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6889
diff
changeset
|
389 |
notify_fixed(fix) |
0 | 390 |
|
391 |
||
4675
9233a8350420
[test] don't display progress bar when testing checkintegrity
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
392 |
def check(repo, cnx, checks, reindex, fix, withpb=True): |
2476
1294a6bdf3bf
application -> instance where it makes sense
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2248
diff
changeset
|
393 |
"""check integrity of instance's repository, |
0 | 394 |
using given user and password to locally connect to the repository |
395 |
(no running cubicweb server needed) |
|
396 |
""" |
|
397 |
# yo, launch checks |
|
398 |
if checks: |
|
399 |
eids_cache = {} |
|
9797
4e640ab62f51
[tests/checkintegrity] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9578
diff
changeset
|
400 |
with cnx.security_enabled(read=False, write=False): # ensure no read security |
4835
13b0b96d7982
[repo] enhanced security handling: deprecates unsafe_execute, in favor of explicit read/write security control using the `enabled_security` context manager. Also code executed on the repository side is now unsafe by default.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4834
diff
changeset
|
401 |
for check in checks: |
13b0b96d7982
[repo] enhanced security handling: deprecates unsafe_execute, in favor of explicit read/write security control using the `enabled_security` context manager. Also code executed on the repository side is now unsafe by default.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4834
diff
changeset
|
402 |
check_func = globals()['check_%s' % check] |
9797
4e640ab62f51
[tests/checkintegrity] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9578
diff
changeset
|
403 |
with cnx.ensure_cnx_set: |
4e640ab62f51
[tests/checkintegrity] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9578
diff
changeset
|
404 |
check_func(repo.schema, cnx, eids_cache, fix=fix) |
0 | 405 |
if fix: |
9797
4e640ab62f51
[tests/checkintegrity] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9578
diff
changeset
|
406 |
cnx.commit() |
0 | 407 |
else: |
408 |
print |
|
409 |
if not fix: |
|
410 |
print 'WARNING: Diagnostic run, nothing has been corrected' |
|
411 |
if reindex: |
|
9797
4e640ab62f51
[tests/checkintegrity] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9578
diff
changeset
|
412 |
cnx.rollback() |
4e640ab62f51
[tests/checkintegrity] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9578
diff
changeset
|
413 |
with cnx.ensure_cnx_set: |
4e640ab62f51
[tests/checkintegrity] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9578
diff
changeset
|
414 |
reindex_entities(repo.schema, cnx, withpb=withpb) |
4e640ab62f51
[tests/checkintegrity] use the new connection api
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9578
diff
changeset
|
415 |
cnx.commit() |