author | Rémi Cardona <remi.cardona@logilab.fr> |
Tue, 15 Sep 2015 18:23:35 +0200 | |
changeset 10682 | 7e111b606005 |
parent 10676 | 9d6b79081bad |
child 10758 | a34edc1057c0 |
permissions | -rw-r--r-- |
9583
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
1 |
# copyright 2003-2014 LOGILAB S.A. (Paris, FRANCE), all rights reserved. |
5421
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5413
diff
changeset
|
2 |
# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5413
diff
changeset
|
3 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5413
diff
changeset
|
4 |
# This file is part of CubicWeb. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5413
diff
changeset
|
5 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5413
diff
changeset
|
6 |
# CubicWeb is free software: you can redistribute it and/or modify it under the |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5413
diff
changeset
|
7 |
# terms of the GNU Lesser General Public License as published by the Free |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5413
diff
changeset
|
8 |
# Software Foundation, either version 2.1 of the License, or (at your option) |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5413
diff
changeset
|
9 |
# any later version. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5413
diff
changeset
|
10 |
# |
5424
8ecbcbff9777
replace logilab-common by CubicWeb in disclaimer
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5421
diff
changeset
|
11 |
# CubicWeb is distributed in the hope that it will be useful, but WITHOUT |
5421
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5413
diff
changeset
|
12 |
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5413
diff
changeset
|
13 |
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5413
diff
changeset
|
14 |
# details. |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5413
diff
changeset
|
15 |
# |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5413
diff
changeset
|
16 |
# You should have received a copy of the GNU Lesser General Public License along |
8167de96c523
proper licensing information (LGPL-2.1). Hope I get it right this time.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5413
diff
changeset
|
17 |
# with CubicWeb. If not, see <http://www.gnu.org/licenses/>. |
0 | 18 |
"""Adapters for native cubicweb sources. |
19 |
||
1952
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
20 |
Notes: |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
21 |
* extid (aka external id, the primary key of an entity in the external source |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
22 |
from which it comes from) are stored in a varchar column encoded as a base64 |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
23 |
string. This is because it should actually be Bytes but we want an index on |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
24 |
it for fast querying. |
5824 | 25 |
""" |
10589
7c23b7de2b8d
[py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10580
diff
changeset
|
26 |
from __future__ import print_function |
7c23b7de2b8d
[py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10580
diff
changeset
|
27 |
|
0 | 28 |
__docformat__ = "restructuredtext en" |
29 |
||
4900 | 30 |
from threading import Lock |
1016
26387b836099
use datetime instead of mx.DateTime
sylvain.thenault@logilab.fr
parents:
973
diff
changeset
|
31 |
from datetime import datetime |
10580
5fbdbbe51867
[server/sources] add decode_extid method
Julien Cristau <julien.cristau@logilab.fr>
parents:
10550
diff
changeset
|
32 |
from base64 import b64encode |
4964
d9e8af8a7a42
[source] implement storages right in the source rather than in hooks
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
4943
diff
changeset
|
33 |
from contextlib import contextmanager |
9468
39b7a91a3f4c
[repo] pylint cleanup, mainly of imports, with a bit of style
Julien Cristau <julien.cristau@logilab.fr>
parents:
9467
diff
changeset
|
34 |
from os.path import basename |
6211
e9d125fd1465
nicer error reporting for unique together constraints
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6201
diff
changeset
|
35 |
import re |
6889
37668bf302f5
improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6850
diff
changeset
|
36 |
import itertools |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
37 |
import zipfile |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
38 |
import logging |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
39 |
import sys |
0 | 40 |
|
10676
9d6b79081bad
[py3k] use text/binary types from six
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10662
diff
changeset
|
41 |
from six import PY2, text_type, binary_type, string_types |
10609
e2d8e81bfe68
[py3k] import range using six.moves
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10602
diff
changeset
|
42 |
from six.moves import range, cPickle as pickle |
10602
4845012cfc8e
[py3k] import 'pickle' using six.moves
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10589
diff
changeset
|
43 |
|
4806
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
44 |
from logilab.common.decorators import cached, clear_cache |
3835
a191b3b9e455
more sensible default values to c-c "create" inputs
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3689
diff
changeset
|
45 |
from logilab.common.configuration import Method |
a191b3b9e455
more sensible default values to c-c "create" inputs
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3689
diff
changeset
|
46 |
from logilab.common.shellutils import getlogin |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
47 |
from logilab.database import get_db_helper, sqlgen |
0 | 48 |
|
6945
28bf94d062a9
[sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6931
diff
changeset
|
49 |
from yams.schema import role_name |
5891
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
50 |
|
6945
28bf94d062a9
[sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6931
diff
changeset
|
51 |
from cubicweb import (UnknownEid, AuthenticationError, ValidationError, Binary, |
10446
1e6655cff5ab
add IUserFriendlyError adapter for violation of check constraints
Julien Cristau <julien.cristau@logilab.fr>
parents:
10411
diff
changeset
|
52 |
UniqueTogetherError, UndoTransactionException, ViolatedConstraint) |
5076
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
53 |
from cubicweb import transaction as tx, server, neg_role |
7954
a3d3220669d6
[cache] replace lgc.cache with something more appropriate (closes #1921713)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
7922
diff
changeset
|
54 |
from cubicweb.utils import QueryCache |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
55 |
from cubicweb.schema import VIRTUAL_RTYPES |
3835
a191b3b9e455
more sensible default values to c-c "create" inputs
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3689
diff
changeset
|
56 |
from cubicweb.cwconfig import CubicWebNoAppConfiguration |
4806
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
57 |
from cubicweb.server import hook |
10200
cceb2c7c02f4
Use our version of schema2sql
Julien Cristau <julien.cristau@logilab.fr>
parents:
10187
diff
changeset
|
58 |
from cubicweb.server import schema2sql as y2sql |
8546
3d2038d6f20d
[sources/native] automatically update passwords using deprecated hashes on login
Julien Cristau <julien.cristau@logilab.fr>
parents:
8520
diff
changeset
|
59 |
from cubicweb.server.utils import crypt_password, eschema_eid, verify_and_update |
2759
23d7a75693f8
R refactor backup and use tar.gz to store all sources
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2638
diff
changeset
|
60 |
from cubicweb.server.sqlutils import SQL_PREFIX, SQLAdapterMixIn |
0 | 61 |
from cubicweb.server.rqlannotation import set_qdata |
5076
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
62 |
from cubicweb.server.hook import CleanupDeletedEidsCacheOp |
7118
e094b3d4eb95
[server] move EditedEntity class to its own module, to avoid cyclic dependency when needed from e.g. session.py
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6957
diff
changeset
|
63 |
from cubicweb.server.edition import EditedEntity |
2625
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2620
diff
changeset
|
64 |
from cubicweb.server.sources import AbstractSource, dbg_st_search, dbg_results |
0 | 65 |
from cubicweb.server.sources.rql2sql import SQLGenerator |
10477
ee21c559f94f
implement a simple statsd logger (closes #5488711)
David Douard <david.douard@logilab.fr>
parents:
10473
diff
changeset
|
66 |
from cubicweb.statsd_logger import statsd_timeit |
0 | 67 |
|
68 |
||
2354
9b4bac626977
ability to map attributes to something else than usual cw mapping on sql generation
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2310
diff
changeset
|
69 |
ATTR_MAP = {} |
0 | 70 |
NONSYSTEM_ETYPES = set() |
71 |
NONSYSTEM_RELATIONS = set() |
|
72 |
||
73 |
class LogCursor(object): |
|
74 |
def __init__(self, cursor): |
|
75 |
self.cu = cursor |
|
1792 | 76 |
|
0 | 77 |
def execute(self, query, args=None): |
78 |
"""Execute a query. |
|
79 |
it's a function just so that it shows up in profiling |
|
80 |
""" |
|
2593
16d9419a4a79
F: start to handle binary debug log level on the server side
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2589
diff
changeset
|
81 |
if server.DEBUG & server.DBG_SQL: |
10589
7c23b7de2b8d
[py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10580
diff
changeset
|
82 |
print('exec', query, args) |
0 | 83 |
try: |
84 |
self.cu.execute(str(query), args) |
|
8695
358d8bed9626
[toward-py3k] rewrite to "except AnException as exc:" (part of #2711624)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
8694
diff
changeset
|
85 |
except Exception as ex: |
10589
7c23b7de2b8d
[py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10580
diff
changeset
|
86 |
print("sql: %r\n args: %s\ndbms message: %r" % ( |
7c23b7de2b8d
[py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10580
diff
changeset
|
87 |
query, args, ex.args[0])) |
0 | 88 |
raise |
1792 | 89 |
|
0 | 90 |
def fetchall(self): |
91 |
return self.cu.fetchall() |
|
1792 | 92 |
|
0 | 93 |
def fetchone(self): |
94 |
return self.cu.fetchone() |
|
1792 | 95 |
|
2625
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2620
diff
changeset
|
96 |
|
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
97 |
def sql_or_clauses(sql, clauses): |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
98 |
select, restr = sql.split(' WHERE ', 1) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
99 |
restrclauses = restr.split(' AND ') |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
100 |
for clause in clauses: |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
101 |
restrclauses.remove(clause) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
102 |
if restrclauses: |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
103 |
restr = '%s AND (%s)' % (' AND '.join(restrclauses), |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
104 |
' OR '.join(clauses)) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
105 |
else: |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
106 |
restr = '(%s)' % ' OR '.join(clauses) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
107 |
return '%s WHERE %s' % (select, restr) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
108 |
|
9450
af4b93bc38a5
[multi-sources-removal] Drop deleted_entities system table and entities.mtime column
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9448
diff
changeset
|
109 |
|
5891
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
110 |
def rdef_table_column(rdef): |
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
111 |
"""return table and column used to store the given relation definition in |
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
112 |
the database |
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
113 |
""" |
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
114 |
return (SQL_PREFIX + str(rdef.subject), |
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
115 |
SQL_PREFIX + str(rdef.rtype)) |
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
116 |
|
9450
af4b93bc38a5
[multi-sources-removal] Drop deleted_entities system table and entities.mtime column
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9448
diff
changeset
|
117 |
|
5891
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
118 |
def rdef_physical_info(dbhelper, rdef): |
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
119 |
"""return backend type and a boolean flag if NULL values should be allowed |
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
120 |
for a given relation definition |
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
121 |
""" |
6359
0bff5a05385c
[sync schema] take care rdef may not be final, in which case we want to use type of eid attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6297
diff
changeset
|
122 |
if rdef.object.final: |
0bff5a05385c
[sync schema] take care rdef may not be final, in which case we want to use type of eid attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6297
diff
changeset
|
123 |
ttype = rdef.object |
0bff5a05385c
[sync schema] take care rdef may not be final, in which case we want to use type of eid attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6297
diff
changeset
|
124 |
else: |
0bff5a05385c
[sync schema] take care rdef may not be final, in which case we want to use type of eid attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6297
diff
changeset
|
125 |
ttype = 'Int' # eid type |
0bff5a05385c
[sync schema] take care rdef may not be final, in which case we want to use type of eid attribute
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6297
diff
changeset
|
126 |
coltype = y2sql.type_from_constraints(dbhelper, ttype, |
5891
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
127 |
rdef.constraints, creating=False) |
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
128 |
allownull = rdef.cardinality[0] != '1' |
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
129 |
return coltype, allownull |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
130 |
|
8265
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
131 |
|
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
132 |
class _UndoException(Exception): |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
133 |
"""something went wrong during undoing""" |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
134 |
|
7530
15178bf89fb6
[server] fix unicode conversion capability in UndoException
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
7507
diff
changeset
|
135 |
def __unicode__(self): |
15178bf89fb6
[server] fix unicode conversion capability in UndoException
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
7507
diff
changeset
|
136 |
"""Called by the unicode builtin; should return a Unicode object |
15178bf89fb6
[server] fix unicode conversion capability in UndoException
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
7507
diff
changeset
|
137 |
|
8265
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
138 |
Type of _UndoException message must be `unicode` by design in CubicWeb. |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
139 |
""" |
10676
9d6b79081bad
[py3k] use text/binary types from six
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10662
diff
changeset
|
140 |
assert isinstance(self.args[0], text_type) |
8265
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
141 |
return self.args[0] |
7530
15178bf89fb6
[server] fix unicode conversion capability in UndoException
Julien Jehannet <julien.jehannet@logilab.fr>
parents:
7507
diff
changeset
|
142 |
|
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
143 |
|
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
144 |
def _undo_check_relation_target(tentity, rdef, role): |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
145 |
"""check linked entity has not been redirected for this relation""" |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
146 |
card = rdef.role_cardinality(role) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
147 |
if card in '?1' and tentity.related(rdef.rtype, role): |
8265
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
148 |
raise _UndoException(tentity._cw._( |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
149 |
"Can't restore %(role)s relation %(rtype)s to entity %(eid)s which " |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
150 |
"is already linked using this relation.") |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
151 |
% {'role': neg_role(role), |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
152 |
'rtype': rdef.rtype, |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
153 |
'eid': tentity.eid}) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
154 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
155 |
def _undo_rel_info(cnx, subj, rtype, obj): |
5076
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
156 |
entities = [] |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
157 |
for role, eid in (('subject', subj), ('object', obj)): |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
158 |
try: |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
159 |
entities.append(cnx.entity_from_eid(eid)) |
5076
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
160 |
except UnknownEid: |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
161 |
raise _UndoException(cnx._( |
5076
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
162 |
"Can't restore relation %(rtype)s, %(role)s entity %(eid)s" |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
163 |
" doesn't exist anymore.") |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
164 |
% {'role': cnx._(role), |
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
165 |
'rtype': cnx._(rtype), |
5076
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
166 |
'eid': eid}) |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
167 |
sentity, oentity = entities |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
168 |
try: |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
169 |
rschema = cnx.vreg.schema.rschema(rtype) |
8900
010a59e12d89
use cw_etype instead of __regid__
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
8821
diff
changeset
|
170 |
rdef = rschema.rdefs[(sentity.cw_etype, oentity.cw_etype)] |
5076
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
171 |
except KeyError: |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
172 |
raise _UndoException(cnx._( |
5076
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
173 |
"Can't restore relation %(rtype)s between %(subj)s and " |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
174 |
"%(obj)s, that relation does not exists anymore in the " |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
175 |
"schema.") |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
176 |
% {'rtype': cnx._(rtype), |
5076
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
177 |
'subj': subj, |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
178 |
'obj': obj}) |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
179 |
return sentity, oentity, rdef |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
180 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
181 |
def _undo_has_later_transaction(cnx, eid): |
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
182 |
return cnx.system_sql('''\ |
5076
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
183 |
SELECT T.tx_uuid FROM transactions AS TREF, transactions AS T |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
184 |
WHERE TREF.tx_uuid='%(txuuid)s' AND T.tx_uuid!='%(txuuid)s' |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
185 |
AND T.tx_time>=TREF.tx_time |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
186 |
AND (EXISTS(SELECT 1 FROM tx_entity_actions AS TEA |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
187 |
WHERE TEA.tx_uuid=T.tx_uuid AND TEA.eid=%(eid)s) |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
188 |
OR EXISTS(SELECT 1 FROM tx_relation_actions as TRA |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
189 |
WHERE TRA.tx_uuid=T.tx_uuid AND ( |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
190 |
TRA.eid_from=%(eid)s OR TRA.eid_to=%(eid)s)) |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
191 |
)''' % {'txuuid': cnx.transaction_data['undoing_uuid'], |
5076
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
192 |
'eid': eid}).fetchone() |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
193 |
|
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
194 |
|
9583
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
195 |
class DefaultEidGenerator(object): |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
196 |
__slots__ = ('source', 'cnx', 'lock') |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
197 |
|
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
198 |
def __init__(self, source): |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
199 |
self.source = source |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
200 |
self.cnx = None |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
201 |
self.lock = Lock() |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
202 |
|
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
203 |
def close(self): |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
204 |
if self.cnx: |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
205 |
self.cnx.close() |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
206 |
self.cnx = None |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
207 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
208 |
def create_eid(self, _cnx, count=1): |
9583
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
209 |
# lock needed to prevent 'Connection is busy with results for another |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
210 |
# command (0)' errors with SQLServer |
9585
3f5b59527d31
[source/native] allow many eid creation per .create_eid call (closes #3526594)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9583
diff
changeset
|
211 |
assert count > 0 |
9583
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
212 |
with self.lock: |
9585
3f5b59527d31
[source/native] allow many eid creation per .create_eid call (closes #3526594)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9583
diff
changeset
|
213 |
return self._create_eid(count) |
9583
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
214 |
|
9585
3f5b59527d31
[source/native] allow many eid creation per .create_eid call (closes #3526594)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9583
diff
changeset
|
215 |
def _create_eid(self, count): |
9583
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
216 |
# internal function doing the eid creation without locking. |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
217 |
# needed for the recursive handling of disconnections (otherwise we |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
218 |
# deadlock on self._eid_cnx_lock |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
219 |
source = self.source |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
220 |
if self.cnx is None: |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
221 |
self.cnx = source.get_connection() |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
222 |
cnx = self.cnx |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
223 |
try: |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
224 |
cursor = cnx.cursor() |
9585
3f5b59527d31
[source/native] allow many eid creation per .create_eid call (closes #3526594)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9583
diff
changeset
|
225 |
for sql in source.dbhelper.sqls_increment_numrange('entities_id_seq', count): |
9583
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
226 |
cursor.execute(sql) |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
227 |
eid = cursor.fetchone()[0] |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
228 |
except (source.OperationalError, source.InterfaceError): |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
229 |
# FIXME: better detection of deconnection pb |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
230 |
source.warning("trying to reconnect create eid connection") |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
231 |
self.cnx = None |
9585
3f5b59527d31
[source/native] allow many eid creation per .create_eid call (closes #3526594)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9583
diff
changeset
|
232 |
return self._create_eid(count) |
9583
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
233 |
except source.DbapiError as exc: |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
234 |
# We get this one with pyodbc and SQL Server when connection was reset |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
235 |
if exc.args[0] == '08S01': |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
236 |
source.warning("trying to reconnect create eid connection") |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
237 |
self.cnx = None |
9585
3f5b59527d31
[source/native] allow many eid creation per .create_eid call (closes #3526594)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9583
diff
changeset
|
238 |
return self._create_eid(count) |
9583
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
239 |
else: |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
240 |
raise |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
241 |
except Exception: # WTF? |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
242 |
cnx.rollback() |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
243 |
self.cnx = None |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
244 |
source.exception('create eid failed in an unforeseen way on SQL statement %s', sql) |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
245 |
raise |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
246 |
else: |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
247 |
cnx.commit() |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
248 |
return eid |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
249 |
|
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
250 |
|
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
251 |
class SQLITEEidGenerator(object): |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
252 |
__slots__ = ('source', 'lock') |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
253 |
|
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
254 |
def __init__(self, source): |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
255 |
self.source = source |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
256 |
self.lock = Lock() |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
257 |
|
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
258 |
def close(self): |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
259 |
pass |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
260 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
261 |
def create_eid(self, cnx, count=1): |
9585
3f5b59527d31
[source/native] allow many eid creation per .create_eid call (closes #3526594)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9583
diff
changeset
|
262 |
assert count > 0 |
9583
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
263 |
source = self.source |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
264 |
with self.lock: |
9585
3f5b59527d31
[source/native] allow many eid creation per .create_eid call (closes #3526594)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9583
diff
changeset
|
265 |
for sql in source.dbhelper.sqls_increment_numrange('entities_id_seq', count): |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
266 |
cursor = source.doexec(cnx, sql) |
9583
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
267 |
return cursor.fetchone()[0] |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
268 |
|
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
269 |
|
0 | 270 |
class NativeSQLSource(SQLAdapterMixIn, AbstractSource): |
271 |
"""adapter for source using the native cubicweb schema (see below) |
|
272 |
""" |
|
2354
9b4bac626977
ability to map attributes to something else than usual cw mapping on sql generation
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2310
diff
changeset
|
273 |
sqlgen_class = SQLGenerator |
0 | 274 |
options = ( |
275 |
('db-driver', |
|
276 |
{'type' : 'string', |
|
277 |
'default': 'postgres', |
|
5423
e15abfdcce38
backport default into stable: stable is now cw 3.8
Sylvain Thénault <sylvain.thenault@logilab.fr>
diff
changeset
|
278 |
# XXX use choice type |
10221
17ad882a211f
[doc] Don't pretend we know cubicweb runs on mysql
Julien Cristau <julien.cristau@logilab.fr>
parents:
10209
diff
changeset
|
279 |
'help': 'database driver (postgres, sqlite, sqlserver2005)', |
6201
cf445fcaf8fe
[c-c create] ask for database driver by default
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
280 |
'group': 'native-source', 'level': 0, |
0 | 281 |
}), |
282 |
('db-host', |
|
283 |
{'type' : 'string', |
|
284 |
'default': '', |
|
285 |
'help': 'database host', |
|
5323
329b4f6d18b4
[config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5217
diff
changeset
|
286 |
'group': 'native-source', 'level': 1, |
0 | 287 |
}), |
2566
714a8743d423
missing db-port option to source's option definitions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2354
diff
changeset
|
288 |
('db-port', |
714a8743d423
missing db-port option to source's option definitions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2354
diff
changeset
|
289 |
{'type' : 'string', |
714a8743d423
missing db-port option to source's option definitions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2354
diff
changeset
|
290 |
'default': '', |
714a8743d423
missing db-port option to source's option definitions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2354
diff
changeset
|
291 |
'help': 'database port', |
5323
329b4f6d18b4
[config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5217
diff
changeset
|
292 |
'group': 'native-source', 'level': 1, |
2566
714a8743d423
missing db-port option to source's option definitions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2354
diff
changeset
|
293 |
}), |
0 | 294 |
('db-name', |
295 |
{'type' : 'string', |
|
3835
a191b3b9e455
more sensible default values to c-c "create" inputs
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3689
diff
changeset
|
296 |
'default': Method('default_instance_id'), |
0 | 297 |
'help': 'database name', |
5323
329b4f6d18b4
[config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5217
diff
changeset
|
298 |
'group': 'native-source', 'level': 0, |
0 | 299 |
}), |
10125
bc6461a7d2da
[server] add a db-namespace option in source definition (closes #1631339)
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
10095
diff
changeset
|
300 |
('db-namespace', |
bc6461a7d2da
[server] add a db-namespace option in source definition (closes #1631339)
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
10095
diff
changeset
|
301 |
{'type' : 'string', |
bc6461a7d2da
[server] add a db-namespace option in source definition (closes #1631339)
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
10095
diff
changeset
|
302 |
'default': '', |
bc6461a7d2da
[server] add a db-namespace option in source definition (closes #1631339)
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
10095
diff
changeset
|
303 |
'help': 'database namespace (schema) name', |
bc6461a7d2da
[server] add a db-namespace option in source definition (closes #1631339)
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
10095
diff
changeset
|
304 |
'group': 'native-source', 'level': 1, |
bc6461a7d2da
[server] add a db-namespace option in source definition (closes #1631339)
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
10095
diff
changeset
|
305 |
}), |
0 | 306 |
('db-user', |
307 |
{'type' : 'string', |
|
3835
a191b3b9e455
more sensible default values to c-c "create" inputs
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3689
diff
changeset
|
308 |
'default': CubicWebNoAppConfiguration.mode == 'user' and getlogin() or 'cubicweb', |
0 | 309 |
'help': 'database user', |
5323
329b4f6d18b4
[config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5217
diff
changeset
|
310 |
'group': 'native-source', 'level': 0, |
0 | 311 |
}), |
312 |
('db-password', |
|
313 |
{'type' : 'password', |
|
314 |
'default': '', |
|
315 |
'help': 'database password', |
|
5323
329b4f6d18b4
[config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5217
diff
changeset
|
316 |
'group': 'native-source', 'level': 0, |
0 | 317 |
}), |
318 |
('db-encoding', |
|
319 |
{'type' : 'string', |
|
320 |
'default': 'utf8', |
|
321 |
'help': 'database encoding', |
|
5323
329b4f6d18b4
[config] with lgc >= 0.50, option's dict inputlevel becomes level
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5217
diff
changeset
|
322 |
'group': 'native-source', 'level': 1, |
0 | 323 |
}), |
5413
dc896e698ab0
added db-extra-arguments to configuration parameters for native source
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
5410
diff
changeset
|
324 |
('db-extra-arguments', |
dc896e698ab0
added db-extra-arguments to configuration parameters for native source
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
5410
diff
changeset
|
325 |
{'type' : 'string', |
dc896e698ab0
added db-extra-arguments to configuration parameters for native source
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
5410
diff
changeset
|
326 |
'default': '', |
dc896e698ab0
added db-extra-arguments to configuration parameters for native source
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
5410
diff
changeset
|
327 |
'help': 'set to "Trusted_Connection" if you are using SQLServer and ' |
dc896e698ab0
added db-extra-arguments to configuration parameters for native source
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
5410
diff
changeset
|
328 |
'want trusted authentication for the database connection', |
5515
513af9be9e37
[config] rename remaining inputlevel to level in option definitions, as expected by lgc >= 0.50
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5426
diff
changeset
|
329 |
'group': 'native-source', 'level': 2, |
5413
dc896e698ab0
added db-extra-arguments to configuration parameters for native source
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
5410
diff
changeset
|
330 |
}), |
9724
e45bf9baa7b7
Add a db-statement-timeout option for postgresql sources
Julien Cristau <julien.cristau@logilab.fr>
parents:
9664
diff
changeset
|
331 |
('db-statement-timeout', |
e45bf9baa7b7
Add a db-statement-timeout option for postgresql sources
Julien Cristau <julien.cristau@logilab.fr>
parents:
9664
diff
changeset
|
332 |
{'type': 'int', |
e45bf9baa7b7
Add a db-statement-timeout option for postgresql sources
Julien Cristau <julien.cristau@logilab.fr>
parents:
9664
diff
changeset
|
333 |
'default': 0, |
e45bf9baa7b7
Add a db-statement-timeout option for postgresql sources
Julien Cristau <julien.cristau@logilab.fr>
parents:
9664
diff
changeset
|
334 |
'help': 'sql statement timeout, in milliseconds (postgres only)', |
e45bf9baa7b7
Add a db-statement-timeout option for postgresql sources
Julien Cristau <julien.cristau@logilab.fr>
parents:
9664
diff
changeset
|
335 |
'group': 'native-source', 'level': 2, |
e45bf9baa7b7
Add a db-statement-timeout option for postgresql sources
Julien Cristau <julien.cristau@logilab.fr>
parents:
9664
diff
changeset
|
336 |
}), |
0 | 337 |
) |
1792 | 338 |
|
6427
c8a5ac2d1eaa
[schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6426
diff
changeset
|
339 |
def __init__(self, repo, source_config, *args, **kwargs): |
9724
e45bf9baa7b7
Add a db-statement-timeout option for postgresql sources
Julien Cristau <julien.cristau@logilab.fr>
parents:
9664
diff
changeset
|
340 |
SQLAdapterMixIn.__init__(self, source_config, repairing=repo.config.repairing) |
3647
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
341 |
self.authentifiers = [LoginPasswordAuthentifier(self)] |
6849
5a0c2cfc19bf
[repository auth] cleanup email login by turning it into a proper repo-side authentication plugin
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6724
diff
changeset
|
342 |
if repo.config['allow-email-login']: |
5a0c2cfc19bf
[repository auth] cleanup email login by turning it into a proper repo-side authentication plugin
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6724
diff
changeset
|
343 |
self.authentifiers.insert(0, EmailPasswordAuthentifier(self)) |
6427
c8a5ac2d1eaa
[schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6426
diff
changeset
|
344 |
AbstractSource.__init__(self, repo, source_config, *args, **kwargs) |
0 | 345 |
# sql generator |
6427
c8a5ac2d1eaa
[schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6426
diff
changeset
|
346 |
self._rql_sqlgen = self.sqlgen_class(self.schema, self.dbhelper, |
4831
c5aec27c1bf7
[repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4829
diff
changeset
|
347 |
ATTR_MAP.copy()) |
0 | 348 |
# full text index helper |
4806
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
349 |
self.do_fti = not repo.config['delay-full-text-indexation'] |
0 | 350 |
# sql queries cache |
7954
a3d3220669d6
[cache] replace lgc.cache with something more appropriate (closes #1921713)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
7922
diff
changeset
|
351 |
self._cache = QueryCache(repo.config['rql-cache-size']) |
4964
d9e8af8a7a42
[source] implement storages right in the source rather than in hooks
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
4943
diff
changeset
|
352 |
# (etype, attr) / storage mapping |
d9e8af8a7a42
[source] implement storages right in the source rather than in hooks
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
4943
diff
changeset
|
353 |
self._storages = {} |
9583
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
354 |
self.binary_to_str = self.dbhelper.dbapi_module.binary_to_str |
5649
a07dee204187
fix unit tests by not using the new create_eid implementation with sqlite
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
5645
diff
changeset
|
355 |
if self.dbdriver == 'sqlite': |
9583
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
356 |
self.eid_generator = SQLITEEidGenerator(self) |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
357 |
else: |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
358 |
self.eid_generator = DefaultEidGenerator(self) |
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
359 |
self.create_eid = self.eid_generator.create_eid |
0 | 360 |
|
6945
28bf94d062a9
[sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6931
diff
changeset
|
361 |
def check_config(self, source_entity): |
28bf94d062a9
[sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6931
diff
changeset
|
362 |
"""check configuration of source entity""" |
28bf94d062a9
[sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6931
diff
changeset
|
363 |
if source_entity.host_config: |
28bf94d062a9
[sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6931
diff
changeset
|
364 |
msg = source_entity._cw._('the system source has its configuration ' |
28bf94d062a9
[sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6931
diff
changeset
|
365 |
'stored on the file-system') |
28bf94d062a9
[sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6931
diff
changeset
|
366 |
raise ValidationError(source_entity.eid, {role_name('config', 'subject'): msg}) |
28bf94d062a9
[sources] refactor source creation and options handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6931
diff
changeset
|
367 |
|
3647
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
368 |
def add_authentifier(self, authentifier): |
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
369 |
self.authentifiers.append(authentifier) |
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
370 |
authentifier.source = self |
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
371 |
authentifier.set_schema(self.schema) |
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
372 |
|
0 | 373 |
def reset_caches(self): |
374 |
"""method called during test to reset potential source caches""" |
|
7954
a3d3220669d6
[cache] replace lgc.cache with something more appropriate (closes #1921713)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
7922
diff
changeset
|
375 |
self._cache = QueryCache(self.repo.config['rql-cache-size']) |
1792 | 376 |
|
0 | 377 |
def clear_eid_cache(self, eid, etype): |
378 |
"""clear potential caches for the given eid""" |
|
2610
2933cc6bf9ad
[F native source] fix attempts to clear cache key
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2593
diff
changeset
|
379 |
self._cache.pop('Any X WHERE X eid %s, X is %s' % (eid, etype), None) |
0 | 380 |
self._cache.pop('Any X WHERE X eid %s' % eid, None) |
2610
2933cc6bf9ad
[F native source] fix attempts to clear cache key
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2593
diff
changeset
|
381 |
self._cache.pop('Any %s' % eid, None) |
1792 | 382 |
|
10477
ee21c559f94f
implement a simple statsd logger (closes #5488711)
David Douard <david.douard@logilab.fr>
parents:
10473
diff
changeset
|
383 |
@statsd_timeit |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
384 |
def sqlexec(self, cnx, sql, args=None): |
0 | 385 |
"""execute the query and return its result""" |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
386 |
return self.process_result(self.doexec(cnx, sql, args)) |
1792 | 387 |
|
7398
26695dd703d8
[repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7342
diff
changeset
|
388 |
def init_creating(self, cnxset=None): |
0 | 389 |
# check full text index availibility |
4806
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
390 |
if self.do_fti: |
7398
26695dd703d8
[repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7342
diff
changeset
|
391 |
if cnxset is None: |
26695dd703d8
[repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7342
diff
changeset
|
392 |
_cnxset = self.repo._get_cnxset() |
6724
24bf6f181d0e
[pyro source] store pyro source mapping file into the database
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6427
diff
changeset
|
393 |
else: |
7398
26695dd703d8
[repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7342
diff
changeset
|
394 |
_cnxset = cnxset |
9463
d62e13eba033
[multi-sources-removal] Simplify ConnectionsSet internal structures and public methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9452
diff
changeset
|
395 |
if not self.dbhelper.has_fti_table(_cnxset.cu): |
4825
cdd979ae1b57
fix name error fixed in wrong branch
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4818
diff
changeset
|
396 |
if not self.repo.config.creating: |
4810
3055dc7a7c88
don't issue critical warning about missing text index when we're creating the instance
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4807
diff
changeset
|
397 |
self.critical('no text index table') |
4806
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
398 |
self.do_fti = False |
7398
26695dd703d8
[repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7342
diff
changeset
|
399 |
if cnxset is None: |
26695dd703d8
[repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7342
diff
changeset
|
400 |
_cnxset.cnxset_freed() |
26695dd703d8
[repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7342
diff
changeset
|
401 |
self.repo._free_cnxset(_cnxset) |
0 | 402 |
|
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
403 |
def backup(self, backupfile, confirm, format='native'): |
2759
23d7a75693f8
R refactor backup and use tar.gz to store all sources
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2638
diff
changeset
|
404 |
"""method called to create a backup of the source's data""" |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
405 |
if format == 'portable': |
8947
3bbd416b09ec
[repo] straightforward bootstrap sequence. Closes #2841188
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8916
diff
changeset
|
406 |
# ensure the schema is the one stored in the database: if repository |
3bbd416b09ec
[repo] straightforward bootstrap sequence. Closes #2841188
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8916
diff
changeset
|
407 |
# started in quick_start mode, the file system's one has been loaded |
3bbd416b09ec
[repo] straightforward bootstrap sequence. Closes #2841188
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8916
diff
changeset
|
408 |
# so force reload |
3bbd416b09ec
[repo] straightforward bootstrap sequence. Closes #2841188
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8916
diff
changeset
|
409 |
if self.repo.config.quick_start: |
3bbd416b09ec
[repo] straightforward bootstrap sequence. Closes #2841188
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8916
diff
changeset
|
410 |
self.repo.set_schema(self.repo.deserialize_schema(), |
3bbd416b09ec
[repo] straightforward bootstrap sequence. Closes #2841188
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8916
diff
changeset
|
411 |
resetvreg=False) |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
412 |
helper = DatabaseIndependentBackupRestore(self) |
7398
26695dd703d8
[repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7342
diff
changeset
|
413 |
self.close_source_connections() |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
414 |
try: |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
415 |
helper.backup(backupfile) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
416 |
finally: |
7398
26695dd703d8
[repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7342
diff
changeset
|
417 |
self.open_source_connections() |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
418 |
elif format == 'native': |
7398
26695dd703d8
[repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7342
diff
changeset
|
419 |
self.close_source_connections() |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
420 |
try: |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
421 |
self.backup_to_file(backupfile, confirm) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
422 |
finally: |
7398
26695dd703d8
[repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7342
diff
changeset
|
423 |
self.open_source_connections() |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
424 |
else: |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
425 |
raise ValueError('Unknown format %r' % format) |
2493
9806571ea790
major refactoring of database dump/restore:
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2476
diff
changeset
|
426 |
|
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
427 |
|
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
428 |
def restore(self, backupfile, confirm, drop, format='native'): |
2493
9806571ea790
major refactoring of database dump/restore:
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2476
diff
changeset
|
429 |
"""method called to restore a backup of source's data""" |
7398
26695dd703d8
[repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7342
diff
changeset
|
430 |
if self.repo.config.init_cnxset_pool: |
26695dd703d8
[repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7342
diff
changeset
|
431 |
self.close_source_connections() |
2759
23d7a75693f8
R refactor backup and use tar.gz to store all sources
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2638
diff
changeset
|
432 |
try: |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
433 |
if format == 'portable': |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
434 |
helper = DatabaseIndependentBackupRestore(self) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
435 |
helper.restore(backupfile) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
436 |
elif format == 'native': |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
437 |
self.restore_from_file(backupfile, confirm, drop=drop) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
438 |
else: |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
439 |
raise ValueError('Unknown format %r' % format) |
2759
23d7a75693f8
R refactor backup and use tar.gz to store all sources
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
2638
diff
changeset
|
440 |
finally: |
7398
26695dd703d8
[repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7342
diff
changeset
|
441 |
if self.repo.config.init_cnxset_pool: |
26695dd703d8
[repository api] definitly kill usage of word 'pool' to refer to connections set used by a session
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7342
diff
changeset
|
442 |
self.open_source_connections() |
2493
9806571ea790
major refactoring of database dump/restore:
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2476
diff
changeset
|
443 |
|
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
444 |
|
6957
ffda12be2e9f
[repository] #1460066: backport datafeed cube as cubicweb source
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6945
diff
changeset
|
445 |
def init(self, activated, source_entity): |
7543
570522300e22
[ms, entity metas] add 'actual source' to entities table / base entity metadata cache. Closes #1767090
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7536
diff
changeset
|
446 |
try: |
7786
18a366267612
[repo] use smarter query to check asource presence on startup (closes #1922099)
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
7754
diff
changeset
|
447 |
# test if 'asource' column exists |
7839
daf46963f4fe
ensure the test for asource column won't crash on SQLServer (closes #1949621)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
7787
diff
changeset
|
448 |
query = self.dbhelper.sql_add_limit_offset('SELECT asource FROM entities', 1) |
daf46963f4fe
ensure the test for asource column won't crash on SQLServer (closes #1949621)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
7787
diff
changeset
|
449 |
source_entity._cw.system_sql(query) |
8695
358d8bed9626
[toward-py3k] rewrite to "except AnException as exc:" (part of #2711624)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
8694
diff
changeset
|
450 |
except Exception as ex: |
7543
570522300e22
[ms, entity metas] add 'actual source' to entities table / base entity metadata cache. Closes #1767090
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7536
diff
changeset
|
451 |
self.eid_type_source = self.eid_type_source_pre_131 |
9131
b3ad80aa645f
fix migration from pre-3.13.1 versions (closes #2846978)
Julien Cristau <julien.cristau@logilab.fr>
parents:
9130
diff
changeset
|
452 |
super(NativeSQLSource, self).init(activated, source_entity) |
b3ad80aa645f
fix migration from pre-3.13.1 versions (closes #2846978)
Julien Cristau <julien.cristau@logilab.fr>
parents:
9130
diff
changeset
|
453 |
self.init_creating(source_entity._cw.cnxset) |
1792 | 454 |
|
5749
b4393b681f7a
[repo] on repository shutdown, we've to close the new eid creation connection
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5729
diff
changeset
|
455 |
def shutdown(self): |
9583
e337a9f658e0
[source/native] refactor eid generation code
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9574
diff
changeset
|
456 |
self.eid_generator.close() |
5749
b4393b681f7a
[repo] on repository shutdown, we've to close the new eid creation connection
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5729
diff
changeset
|
457 |
|
9492
c7fc56eecd1a
English typography
Dimitri Papadopoulos <dimitri.papadopoulos@cea.fr>
parents:
9375
diff
changeset
|
458 |
# XXX deprecates [un]map_attribute? |
5013
ad91f93bbb93
[source storage] refactor source sql generation and results handling to allow repository side callbacks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5011
diff
changeset
|
459 |
def map_attribute(self, etype, attr, cb, sourcedb=True): |
10095
200bd6a601dc
[warnings] put an end to warnings in the sqlite driver over `str` being sent instead of unicode strings
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10086
diff
changeset
|
460 |
self._rql_sqlgen.attr_map[u'%s.%s' % (etype, attr)] = (cb, sourcedb) |
2493
9806571ea790
major refactoring of database dump/restore:
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2476
diff
changeset
|
461 |
|
4512
e7ac20bf3629
unset_attribute_storage, for testing purpose at least
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
462 |
def unmap_attribute(self, etype, attr): |
10095
200bd6a601dc
[warnings] put an end to warnings in the sqlite driver over `str` being sent instead of unicode strings
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10086
diff
changeset
|
463 |
self._rql_sqlgen.attr_map.pop(u'%s.%s' % (etype, attr), None) |
4512
e7ac20bf3629
unset_attribute_storage, for testing purpose at least
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4252
diff
changeset
|
464 |
|
4964
d9e8af8a7a42
[source] implement storages right in the source rather than in hooks
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
4943
diff
changeset
|
465 |
def set_storage(self, etype, attr, storage): |
d9e8af8a7a42
[source] implement storages right in the source rather than in hooks
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
4943
diff
changeset
|
466 |
storage_dict = self._storages.setdefault(etype, {}) |
d9e8af8a7a42
[source] implement storages right in the source rather than in hooks
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
4943
diff
changeset
|
467 |
storage_dict[attr] = storage |
5013
ad91f93bbb93
[source storage] refactor source sql generation and results handling to allow repository side callbacks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5011
diff
changeset
|
468 |
self.map_attribute(etype, attr, |
ad91f93bbb93
[source storage] refactor source sql generation and results handling to allow repository side callbacks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5011
diff
changeset
|
469 |
storage.callback, storage.is_source_callback) |
4964
d9e8af8a7a42
[source] implement storages right in the source rather than in hooks
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
4943
diff
changeset
|
470 |
|
d9e8af8a7a42
[source] implement storages right in the source rather than in hooks
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
4943
diff
changeset
|
471 |
def unset_storage(self, etype, attr): |
d9e8af8a7a42
[source] implement storages right in the source rather than in hooks
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
4943
diff
changeset
|
472 |
self._storages[etype].pop(attr) |
d9e8af8a7a42
[source] implement storages right in the source rather than in hooks
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
4943
diff
changeset
|
473 |
# if etype has no storage left, remove the entry |
d9e8af8a7a42
[source] implement storages right in the source rather than in hooks
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
4943
diff
changeset
|
474 |
if not self._storages[etype]: |
d9e8af8a7a42
[source] implement storages right in the source rather than in hooks
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
4943
diff
changeset
|
475 |
del self._storages[etype] |
d9e8af8a7a42
[source] implement storages right in the source rather than in hooks
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
4943
diff
changeset
|
476 |
self.unmap_attribute(etype, attr) |
d9e8af8a7a42
[source] implement storages right in the source rather than in hooks
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
4943
diff
changeset
|
477 |
|
5397
cdbf823450aa
[bfss] new storage_changed migration action to move an attribute to a custom storage. Closes #893941
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5323
diff
changeset
|
478 |
def storage(self, etype, attr): |
cdbf823450aa
[bfss] new storage_changed migration action to move an attribute to a custom storage. Closes #893941
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5323
diff
changeset
|
479 |
"""return the storage for the given entity type / attribute |
cdbf823450aa
[bfss] new storage_changed migration action to move an attribute to a custom storage. Closes #893941
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5323
diff
changeset
|
480 |
""" |
cdbf823450aa
[bfss] new storage_changed migration action to move an attribute to a custom storage. Closes #893941
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5323
diff
changeset
|
481 |
try: |
cdbf823450aa
[bfss] new storage_changed migration action to move an attribute to a custom storage. Closes #893941
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5323
diff
changeset
|
482 |
return self._storages[etype][attr] |
cdbf823450aa
[bfss] new storage_changed migration action to move an attribute to a custom storage. Closes #893941
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5323
diff
changeset
|
483 |
except KeyError: |
cdbf823450aa
[bfss] new storage_changed migration action to move an attribute to a custom storage. Closes #893941
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5323
diff
changeset
|
484 |
raise Exception('no custom storage set for %s.%s' % (etype, attr)) |
cdbf823450aa
[bfss] new storage_changed migration action to move an attribute to a custom storage. Closes #893941
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5323
diff
changeset
|
485 |
|
0 | 486 |
# ISource interface ####################################################### |
487 |
||
10477
ee21c559f94f
implement a simple statsd logger (closes #5488711)
David Douard <david.douard@logilab.fr>
parents:
10473
diff
changeset
|
488 |
@statsd_timeit |
3647
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
489 |
def compile_rql(self, rql, sols): |
3240
8604a15995d1
refactor so that rql rewriter may be used outside the server. Enhance it to be usable for RRQLExpression as well
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3041
diff
changeset
|
490 |
rqlst = self.repo.vreg.rqlhelper.parse(rql) |
0 | 491 |
rqlst.restricted_vars = () |
3647
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
492 |
rqlst.children[0].solutions = sols |
0 | 493 |
self.repo.querier.sqlgen_annotate(rqlst) |
438 | 494 |
set_qdata(self.schema.rschema, rqlst, ()) |
0 | 495 |
return rqlst |
1792 | 496 |
|
0 | 497 |
def set_schema(self, schema): |
2476
1294a6bdf3bf
application -> instance where it makes sense
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2354
diff
changeset
|
498 |
"""set the instance'schema""" |
7954
a3d3220669d6
[cache] replace lgc.cache with something more appropriate (closes #1921713)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
7922
diff
changeset
|
499 |
self._cache = QueryCache(self.repo.config['rql-cache-size']) |
0 | 500 |
self.cache_hit, self.cache_miss, self.no_cache = 0, 0, 0 |
501 |
self.schema = schema |
|
502 |
try: |
|
503 |
self._rql_sqlgen.schema = schema |
|
504 |
except AttributeError: |
|
505 |
pass # __init__ |
|
3647
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
506 |
for authentifier in self.authentifiers: |
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
507 |
authentifier.set_schema(self.schema) |
4806
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
508 |
clear_cache(self, 'need_fti_indexation') |
1792 | 509 |
|
0 | 510 |
def support_entity(self, etype, write=False): |
511 |
"""return true if the given entity's type is handled by this adapter |
|
512 |
if write is true, return true only if it's a RW support |
|
513 |
""" |
|
514 |
return not etype in NONSYSTEM_ETYPES |
|
1792 | 515 |
|
0 | 516 |
def support_relation(self, rtype, write=False): |
517 |
"""return true if the given relation's type is handled by this adapter |
|
518 |
if write is true, return true only if it's a RW support |
|
519 |
""" |
|
520 |
if write: |
|
521 |
return not rtype in NONSYSTEM_RELATIONS |
|
522 |
# due to current multi-sources implementation, the system source |
|
1792 | 523 |
# can't claim not supporting a relation |
0 | 524 |
return True #not rtype == 'content_for' |
525 |
||
10477
ee21c559f94f
implement a simple statsd logger (closes #5488711)
David Douard <david.douard@logilab.fr>
parents:
10473
diff
changeset
|
526 |
@statsd_timeit |
9512
88dc96fc9fc1
[server] use a connection instead of a session for user authentication
Julien Cristau <julien.cristau@logilab.fr>
parents:
9487
diff
changeset
|
527 |
def authenticate(self, cnx, login, **kwargs): |
3647
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
528 |
"""return CWUser eid for the given login and other authentication |
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
529 |
information found in kwargs, else raise `AuthenticationError` |
0 | 530 |
""" |
3647
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
531 |
for authentifier in self.authentifiers: |
0 | 532 |
try: |
9512
88dc96fc9fc1
[server] use a connection instead of a session for user authentication
Julien Cristau <julien.cristau@logilab.fr>
parents:
9487
diff
changeset
|
533 |
return authentifier.authenticate(cnx, login, **kwargs) |
3647
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
534 |
except AuthenticationError: |
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
535 |
continue |
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
536 |
raise AuthenticationError() |
1792 | 537 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
538 |
def syntax_tree_search(self, cnx, union, args=None, cachekey=None, |
0 | 539 |
varmap=None): |
540 |
"""return result from this source for a rql query (actually from |
|
541 |
a rql syntax tree and a solution dictionary mapping each used |
|
542 |
variable to a possible type). If cachekey is given, the query |
|
543 |
necessary to fetch the results (but not the results themselves) |
|
544 |
may be cached using this key. |
|
545 |
""" |
|
2625
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2620
diff
changeset
|
546 |
assert dbg_st_search(self.uri, union, varmap, args, cachekey) |
0 | 547 |
# remember number of actually selected term (sql generation may append some) |
548 |
if cachekey is None: |
|
549 |
self.no_cache += 1 |
|
550 |
# generate sql query if we are able to do so (not supported types...) |
|
5013
ad91f93bbb93
[source storage] refactor source sql generation and results handling to allow repository side callbacks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5011
diff
changeset
|
551 |
sql, qargs, cbs = self._rql_sqlgen.generate(union, args, varmap) |
0 | 552 |
else: |
553 |
# sql may be cached |
|
554 |
try: |
|
5013
ad91f93bbb93
[source storage] refactor source sql generation and results handling to allow repository side callbacks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5011
diff
changeset
|
555 |
sql, qargs, cbs = self._cache[cachekey] |
0 | 556 |
self.cache_hit += 1 |
557 |
except KeyError: |
|
558 |
self.cache_miss += 1 |
|
5013
ad91f93bbb93
[source storage] refactor source sql generation and results handling to allow repository side callbacks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5011
diff
changeset
|
559 |
sql, qargs, cbs = self._rql_sqlgen.generate(union, args, varmap) |
ad91f93bbb93
[source storage] refactor source sql generation and results handling to allow repository side callbacks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5011
diff
changeset
|
560 |
self._cache[cachekey] = sql, qargs, cbs |
ad91f93bbb93
[source storage] refactor source sql generation and results handling to allow repository side callbacks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5011
diff
changeset
|
561 |
args = self.merge_args(args, qargs) |
10612
84468b90e9c1
[py3k] basestring → six.string_types
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10609
diff
changeset
|
562 |
assert isinstance(sql, string_types), repr(sql) |
10366
38c7598b5c61
[connection] remove the `mode` attribute
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10365
diff
changeset
|
563 |
cursor = self.doexec(cnx, sql, args) |
10086
98bc2ca1a816
[source/native] session -> cnx
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10074
diff
changeset
|
564 |
results = self.process_result(cursor, cnx, cbs) |
2625
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2620
diff
changeset
|
565 |
assert dbg_results(results) |
d6012db7b93e
R [server debug] more server side debugging tweaks
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2620
diff
changeset
|
566 |
return results |
1792 | 567 |
|
4964
d9e8af8a7a42
[source] implement storages right in the source rather than in hooks
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
4943
diff
changeset
|
568 |
@contextmanager |
10550
d4bd28d5fca8
[server/sources] make sure entity._cw is a Connection before calling Storages
Julien Cristau <julien.cristau@logilab.fr>
parents:
10477
diff
changeset
|
569 |
def _fixup_cw(self, cnx, entity): |
d4bd28d5fca8
[server/sources] make sure entity._cw is a Connection before calling Storages
Julien Cristau <julien.cristau@logilab.fr>
parents:
10477
diff
changeset
|
570 |
_cw = entity._cw |
d4bd28d5fca8
[server/sources] make sure entity._cw is a Connection before calling Storages
Julien Cristau <julien.cristau@logilab.fr>
parents:
10477
diff
changeset
|
571 |
entity._cw = cnx |
d4bd28d5fca8
[server/sources] make sure entity._cw is a Connection before calling Storages
Julien Cristau <julien.cristau@logilab.fr>
parents:
10477
diff
changeset
|
572 |
try: |
d4bd28d5fca8
[server/sources] make sure entity._cw is a Connection before calling Storages
Julien Cristau <julien.cristau@logilab.fr>
parents:
10477
diff
changeset
|
573 |
yield |
d4bd28d5fca8
[server/sources] make sure entity._cw is a Connection before calling Storages
Julien Cristau <julien.cristau@logilab.fr>
parents:
10477
diff
changeset
|
574 |
finally: |
d4bd28d5fca8
[server/sources] make sure entity._cw is a Connection before calling Storages
Julien Cristau <julien.cristau@logilab.fr>
parents:
10477
diff
changeset
|
575 |
entity._cw = _cw |
d4bd28d5fca8
[server/sources] make sure entity._cw is a Connection before calling Storages
Julien Cristau <julien.cristau@logilab.fr>
parents:
10477
diff
changeset
|
576 |
|
d4bd28d5fca8
[server/sources] make sure entity._cw is a Connection before calling Storages
Julien Cristau <julien.cristau@logilab.fr>
parents:
10477
diff
changeset
|
577 |
@contextmanager |
d4bd28d5fca8
[server/sources] make sure entity._cw is a Connection before calling Storages
Julien Cristau <julien.cristau@logilab.fr>
parents:
10477
diff
changeset
|
578 |
def _storage_handler(self, cnx, entity, event): |
4964
d9e8af8a7a42
[source] implement storages right in the source rather than in hooks
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
4943
diff
changeset
|
579 |
# 1/ memorize values as they are before the storage is called. |
d9e8af8a7a42
[source] implement storages right in the source rather than in hooks
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
4943
diff
changeset
|
580 |
# For instance, the BFSStorage will replace the `data` |
d9e8af8a7a42
[source] implement storages right in the source rather than in hooks
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
4943
diff
changeset
|
581 |
# binary value with a Binary containing the destination path |
d9e8af8a7a42
[source] implement storages right in the source rather than in hooks
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
4943
diff
changeset
|
582 |
# on the filesystem. To make the entity.data usage absolutely |
d9e8af8a7a42
[source] implement storages right in the source rather than in hooks
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
4943
diff
changeset
|
583 |
# transparent, we'll have to reset entity.data to its binary |
d9e8af8a7a42
[source] implement storages right in the source rather than in hooks
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
4943
diff
changeset
|
584 |
# value once the SQL query will be executed |
6889
37668bf302f5
improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6850
diff
changeset
|
585 |
restore_values = [] |
37668bf302f5
improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6850
diff
changeset
|
586 |
if isinstance(entity, list): |
37668bf302f5
improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6850
diff
changeset
|
587 |
entities = entity |
37668bf302f5
improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6850
diff
changeset
|
588 |
else: |
37668bf302f5
improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6850
diff
changeset
|
589 |
entities = [entity] |
37668bf302f5
improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6850
diff
changeset
|
590 |
etype = entities[0].__regid__ |
4964
d9e8af8a7a42
[source] implement storages right in the source rather than in hooks
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
4943
diff
changeset
|
591 |
for attr, storage in self._storages.get(etype, {}).items(): |
6889
37668bf302f5
improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6850
diff
changeset
|
592 |
for entity in entities: |
10550
d4bd28d5fca8
[server/sources] make sure entity._cw is a Connection before calling Storages
Julien Cristau <julien.cristau@logilab.fr>
parents:
10477
diff
changeset
|
593 |
with self._fixup_cw(cnx, entity): |
d4bd28d5fca8
[server/sources] make sure entity._cw is a Connection before calling Storages
Julien Cristau <julien.cristau@logilab.fr>
parents:
10477
diff
changeset
|
594 |
if event == 'deleted': |
d4bd28d5fca8
[server/sources] make sure entity._cw is a Connection before calling Storages
Julien Cristau <julien.cristau@logilab.fr>
parents:
10477
diff
changeset
|
595 |
storage.entity_deleted(entity, attr) |
d4bd28d5fca8
[server/sources] make sure entity._cw is a Connection before calling Storages
Julien Cristau <julien.cristau@logilab.fr>
parents:
10477
diff
changeset
|
596 |
else: |
d4bd28d5fca8
[server/sources] make sure entity._cw is a Connection before calling Storages
Julien Cristau <julien.cristau@logilab.fr>
parents:
10477
diff
changeset
|
597 |
edited = entity.cw_edited |
d4bd28d5fca8
[server/sources] make sure entity._cw is a Connection before calling Storages
Julien Cristau <julien.cristau@logilab.fr>
parents:
10477
diff
changeset
|
598 |
if attr in edited: |
d4bd28d5fca8
[server/sources] make sure entity._cw is a Connection before calling Storages
Julien Cristau <julien.cristau@logilab.fr>
parents:
10477
diff
changeset
|
599 |
handler = getattr(storage, 'entity_%s' % event) |
d4bd28d5fca8
[server/sources] make sure entity._cw is a Connection before calling Storages
Julien Cristau <julien.cristau@logilab.fr>
parents:
10477
diff
changeset
|
600 |
to_restore = handler(entity, attr) |
d4bd28d5fca8
[server/sources] make sure entity._cw is a Connection before calling Storages
Julien Cristau <julien.cristau@logilab.fr>
parents:
10477
diff
changeset
|
601 |
restore_values.append((entity, attr, to_restore)) |
5128
e5d300d75519
[python] take care to this detail of @contextmanager: if an unhandled exception occurs in the block, it is reraised inside the generator at the point where the yield occurred
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5107
diff
changeset
|
602 |
try: |
e5d300d75519
[python] take care to this detail of @contextmanager: if an unhandled exception occurs in the block, it is reraised inside the generator at the point where the yield occurred
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5107
diff
changeset
|
603 |
yield # 2/ execute the source's instructions |
e5d300d75519
[python] take care to this detail of @contextmanager: if an unhandled exception occurs in the block, it is reraised inside the generator at the point where the yield occurred
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5107
diff
changeset
|
604 |
finally: |
e5d300d75519
[python] take care to this detail of @contextmanager: if an unhandled exception occurs in the block, it is reraised inside the generator at the point where the yield occurred
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5107
diff
changeset
|
605 |
# 3/ restore original values |
6889
37668bf302f5
improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6850
diff
changeset
|
606 |
for entity, attr, value in restore_values: |
6142
8bc6eac1fac1
[session] cleanup hook / operation / entity edition api
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6128
diff
changeset
|
607 |
entity.cw_edited.edited_attribute(attr, value) |
4964
d9e8af8a7a42
[source] implement storages right in the source rather than in hooks
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
4943
diff
changeset
|
608 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
609 |
def add_entity(self, cnx, entity): |
0 | 610 |
"""add a new entity to the source""" |
10550
d4bd28d5fca8
[server/sources] make sure entity._cw is a Connection before calling Storages
Julien Cristau <julien.cristau@logilab.fr>
parents:
10477
diff
changeset
|
611 |
with self._storage_handler(cnx, entity, 'added'): |
4964
d9e8af8a7a42
[source] implement storages right in the source rather than in hooks
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
4943
diff
changeset
|
612 |
attrs = self.preprocess_entity(entity) |
8900
010a59e12d89
use cw_etype instead of __regid__
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
8821
diff
changeset
|
613 |
sql = self.sqlgen.insert(SQL_PREFIX + entity.cw_etype, attrs) |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
614 |
self.doexec(cnx, sql, attrs) |
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
615 |
if cnx.ertype_supports_undo(entity.cw_etype): |
10264
d5403de0792b
[server] use unicode wherever appropriate in transaction/undo related methods
Julien Cristau <julien.cristau@logilab.fr>
parents:
10221
diff
changeset
|
616 |
self._record_tx_action(cnx, 'tx_entity_actions', u'C', |
10676
9d6b79081bad
[py3k] use text/binary types from six
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10662
diff
changeset
|
617 |
etype=text_type(entity.cw_etype), eid=entity.eid) |
1792 | 618 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
619 |
def update_entity(self, cnx, entity): |
0 | 620 |
"""replace an entity in the source""" |
10550
d4bd28d5fca8
[server/sources] make sure entity._cw is a Connection before calling Storages
Julien Cristau <julien.cristau@logilab.fr>
parents:
10477
diff
changeset
|
621 |
with self._storage_handler(cnx, entity, 'updated'): |
4964
d9e8af8a7a42
[source] implement storages right in the source rather than in hooks
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
4943
diff
changeset
|
622 |
attrs = self.preprocess_entity(entity) |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
623 |
if cnx.ertype_supports_undo(entity.cw_etype): |
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
624 |
changes = self._save_attrs(cnx, entity, attrs) |
10264
d5403de0792b
[server] use unicode wherever appropriate in transaction/undo related methods
Julien Cristau <julien.cristau@logilab.fr>
parents:
10221
diff
changeset
|
625 |
self._record_tx_action(cnx, 'tx_entity_actions', u'U', |
10676
9d6b79081bad
[py3k] use text/binary types from six
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10662
diff
changeset
|
626 |
etype=text_type(entity.cw_etype), eid=entity.eid, |
10602
4845012cfc8e
[py3k] import 'pickle' using six.moves
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10589
diff
changeset
|
627 |
changes=self._binary(pickle.dumps(changes))) |
8900
010a59e12d89
use cw_etype instead of __regid__
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
8821
diff
changeset
|
628 |
sql = self.sqlgen.update(SQL_PREFIX + entity.cw_etype, attrs, |
4964
d9e8af8a7a42
[source] implement storages right in the source rather than in hooks
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
4943
diff
changeset
|
629 |
['cw_eid']) |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
630 |
self.doexec(cnx, sql, attrs) |
0 | 631 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
632 |
def delete_entity(self, cnx, entity): |
0 | 633 |
"""delete an entity from the source""" |
10550
d4bd28d5fca8
[server/sources] make sure entity._cw is a Connection before calling Storages
Julien Cristau <julien.cristau@logilab.fr>
parents:
10477
diff
changeset
|
634 |
with self._storage_handler(cnx, entity, 'deleted'): |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
635 |
if cnx.ertype_supports_undo(entity.cw_etype): |
4964
d9e8af8a7a42
[source] implement storages right in the source rather than in hooks
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
4943
diff
changeset
|
636 |
attrs = [SQL_PREFIX + r.type |
d9e8af8a7a42
[source] implement storages right in the source rather than in hooks
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
4943
diff
changeset
|
637 |
for r in entity.e_schema.subject_relations() |
d9e8af8a7a42
[source] implement storages right in the source rather than in hooks
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
4943
diff
changeset
|
638 |
if (r.final or r.inlined) and not r in VIRTUAL_RTYPES] |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
639 |
changes = self._save_attrs(cnx, entity, attrs) |
10264
d5403de0792b
[server] use unicode wherever appropriate in transaction/undo related methods
Julien Cristau <julien.cristau@logilab.fr>
parents:
10221
diff
changeset
|
640 |
self._record_tx_action(cnx, 'tx_entity_actions', u'D', |
10676
9d6b79081bad
[py3k] use text/binary types from six
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10662
diff
changeset
|
641 |
etype=text_type(entity.cw_etype), eid=entity.eid, |
10602
4845012cfc8e
[py3k] import 'pickle' using six.moves
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10589
diff
changeset
|
642 |
changes=self._binary(pickle.dumps(changes))) |
4964
d9e8af8a7a42
[source] implement storages right in the source rather than in hooks
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
4943
diff
changeset
|
643 |
attrs = {'cw_eid': entity.eid} |
8900
010a59e12d89
use cw_etype instead of __regid__
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
8821
diff
changeset
|
644 |
sql = self.sqlgen.delete(SQL_PREFIX + entity.cw_etype, attrs) |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
645 |
self.doexec(cnx, sql, attrs) |
0 | 646 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
647 |
def add_relation(self, cnx, subject, rtype, object, inlined=False): |
5072
072ae171aeb0
[cleanup] style fixes, add nodes, 0.2 cents refactorings
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5071
diff
changeset
|
648 |
"""add a relation to the source""" |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
649 |
self._add_relations(cnx, rtype, [(subject, object)], inlined) |
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
650 |
if cnx.ertype_supports_undo(rtype): |
10264
d5403de0792b
[server] use unicode wherever appropriate in transaction/undo related methods
Julien Cristau <julien.cristau@logilab.fr>
parents:
10221
diff
changeset
|
651 |
self._record_tx_action(cnx, 'tx_relation_actions', u'A', |
10676
9d6b79081bad
[py3k] use text/binary types from six
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10662
diff
changeset
|
652 |
eid_from=subject, rtype=text_type(rtype), eid_to=object) |
5072
072ae171aeb0
[cleanup] style fixes, add nodes, 0.2 cents refactorings
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5071
diff
changeset
|
653 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
654 |
def add_relations(self, cnx, rtype, subj_obj_list, inlined=False): |
7237
9f619715665b
[server] improve the speed of setting relations between entities (closes #1625257)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
7202
diff
changeset
|
655 |
"""add a relations to the source""" |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
656 |
self._add_relations(cnx, rtype, subj_obj_list, inlined) |
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
657 |
if cnx.ertype_supports_undo(rtype): |
7237
9f619715665b
[server] improve the speed of setting relations between entities (closes #1625257)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
7202
diff
changeset
|
658 |
for subject, object in subj_obj_list: |
10264
d5403de0792b
[server] use unicode wherever appropriate in transaction/undo related methods
Julien Cristau <julien.cristau@logilab.fr>
parents:
10221
diff
changeset
|
659 |
self._record_tx_action(cnx, 'tx_relation_actions', u'A', |
10676
9d6b79081bad
[py3k] use text/binary types from six
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10662
diff
changeset
|
660 |
eid_from=subject, rtype=text_type(rtype), eid_to=object) |
7551 | 661 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
662 |
def _add_relations(self, cnx, rtype, subj_obj_list, inlined=False): |
0 | 663 |
"""add a relation to the source""" |
7243
9ab01bf84eac
fix bad handling of inlined relations when entities of different etypes are passed as subject to add_relations
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
7237
diff
changeset
|
664 |
sql = [] |
4818
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4810
diff
changeset
|
665 |
if inlined is False: |
7237
9f619715665b
[server] improve the speed of setting relations between entities (closes #1625257)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
7202
diff
changeset
|
666 |
attrs = [{'eid_from': subject, 'eid_to': object} |
9f619715665b
[server] improve the speed of setting relations between entities (closes #1625257)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
7202
diff
changeset
|
667 |
for subject, object in subj_obj_list] |
7243
9ab01bf84eac
fix bad handling of inlined relations when entities of different etypes are passed as subject to add_relations
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
7237
diff
changeset
|
668 |
sql.append((self.sqlgen.insert('%s_relation' % rtype, attrs[0]), attrs)) |
4818
9f9bfbcdecfd
le patch massiveimport a été importé
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4810
diff
changeset
|
669 |
else: # used by data import |
7243
9ab01bf84eac
fix bad handling of inlined relations when entities of different etypes are passed as subject to add_relations
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
7237
diff
changeset
|
670 |
etypes = {} |
9ab01bf84eac
fix bad handling of inlined relations when entities of different etypes are passed as subject to add_relations
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
7237
diff
changeset
|
671 |
for subject, object in subj_obj_list: |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
672 |
etype = cnx.entity_metas(subject)['type'] |
7243
9ab01bf84eac
fix bad handling of inlined relations when entities of different etypes are passed as subject to add_relations
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
7237
diff
changeset
|
673 |
if etype in etypes: |
9ab01bf84eac
fix bad handling of inlined relations when entities of different etypes are passed as subject to add_relations
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
7237
diff
changeset
|
674 |
etypes[etype].append((subject, object)) |
9ab01bf84eac
fix bad handling of inlined relations when entities of different etypes are passed as subject to add_relations
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
7237
diff
changeset
|
675 |
else: |
9ab01bf84eac
fix bad handling of inlined relations when entities of different etypes are passed as subject to add_relations
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
7237
diff
changeset
|
676 |
etypes[etype] = [(subject, object)] |
10662
10942ed172de
[py3k] dict.iteritems → dict.items
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10651
diff
changeset
|
677 |
for subj_etype, subj_obj_list in etypes.items(): |
7243
9ab01bf84eac
fix bad handling of inlined relations when entities of different etypes are passed as subject to add_relations
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
7237
diff
changeset
|
678 |
attrs = [{'cw_eid': subject, SQL_PREFIX + rtype: object} |
9ab01bf84eac
fix bad handling of inlined relations when entities of different etypes are passed as subject to add_relations
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
7237
diff
changeset
|
679 |
for subject, object in subj_obj_list] |
9ab01bf84eac
fix bad handling of inlined relations when entities of different etypes are passed as subject to add_relations
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
7237
diff
changeset
|
680 |
sql.append((self.sqlgen.update(SQL_PREFIX + etype, attrs[0], |
9ab01bf84eac
fix bad handling of inlined relations when entities of different etypes are passed as subject to add_relations
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
7237
diff
changeset
|
681 |
['cw_eid']), |
9ab01bf84eac
fix bad handling of inlined relations when entities of different etypes are passed as subject to add_relations
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
7237
diff
changeset
|
682 |
attrs)) |
9ab01bf84eac
fix bad handling of inlined relations when entities of different etypes are passed as subject to add_relations
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
7237
diff
changeset
|
683 |
for statement, attrs in sql: |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
684 |
self.doexecmany(cnx, statement, attrs) |
1792 | 685 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
686 |
def delete_relation(self, cnx, subject, rtype, object): |
0 | 687 |
"""delete a relation from the source""" |
688 |
rschema = self.schema.rschema(rtype) |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
689 |
self._delete_relation(cnx, subject, rtype, object, rschema.inlined) |
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
690 |
if cnx.ertype_supports_undo(rtype): |
10264
d5403de0792b
[server] use unicode wherever appropriate in transaction/undo related methods
Julien Cristau <julien.cristau@logilab.fr>
parents:
10221
diff
changeset
|
691 |
self._record_tx_action(cnx, 'tx_relation_actions', u'R', |
10676
9d6b79081bad
[py3k] use text/binary types from six
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10662
diff
changeset
|
692 |
eid_from=subject, rtype=text_type(rtype), eid_to=object) |
5076
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
693 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
694 |
def _delete_relation(self, cnx, subject, rtype, object, inlined=False): |
5076
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
695 |
"""delete a relation from the source""" |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
696 |
if inlined: |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
697 |
table = SQL_PREFIX + cnx.entity_metas(subject)['type'] |
1251
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1079
diff
changeset
|
698 |
column = SQL_PREFIX + rtype |
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1079
diff
changeset
|
699 |
sql = 'UPDATE %s SET %s=NULL WHERE %seid=%%(eid)s' % (table, column, |
af40e615dc89
introduce a 'cw_' prefix on entity table and column names so we don't conflict with sql or DBMS specific keywords
sylvain.thenault@logilab.fr
parents:
1079
diff
changeset
|
700 |
SQL_PREFIX) |
0 | 701 |
attrs = {'eid' : subject} |
702 |
else: |
|
703 |
attrs = {'eid_from': subject, 'eid_to': object} |
|
704 |
sql = self.sqlgen.delete('%s_relation' % rtype, attrs) |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
705 |
self.doexec(cnx, sql, attrs) |
0 | 706 |
|
10477
ee21c559f94f
implement a simple statsd logger (closes #5488711)
David Douard <david.douard@logilab.fr>
parents:
10473
diff
changeset
|
707 |
@statsd_timeit |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
708 |
def doexec(self, cnx, query, args=None, rollback=True): |
0 | 709 |
"""Execute a query. |
710 |
it's a function just so that it shows up in profiling |
|
711 |
""" |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
712 |
cursor = cnx.cnxset.cu |
2593
16d9419a4a79
F: start to handle binary debug log level on the server side
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2589
diff
changeset
|
713 |
if server.DEBUG & server.DBG_SQL: |
10589
7c23b7de2b8d
[py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10580
diff
changeset
|
714 |
print('exec', query, args, cnx.cnxset.cnx) |
0 | 715 |
try: |
9701
46c8d8701240
an unicode string -> a unicode string
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9492
diff
changeset
|
716 |
# str(query) to avoid error if it's a unicode string |
0 | 717 |
cursor.execute(str(query), args) |
8695
358d8bed9626
[toward-py3k] rewrite to "except AnException as exc:" (part of #2711624)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
8694
diff
changeset
|
718 |
except Exception as ex: |
4682
4994901b7379
don't issue critical message when trying to alter sqlite db during test
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4517
diff
changeset
|
719 |
if self.repo.config.mode != 'test': |
4994901b7379
don't issue critical message when trying to alter sqlite db during test
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4517
diff
changeset
|
720 |
# during test we get those message when trying to alter sqlite |
4994901b7379
don't issue critical message when trying to alter sqlite db during test
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4517
diff
changeset
|
721 |
# db schema |
10178
4b33246cb43c
[native] Change SQL errors log level.
Christophe de Vienne <christophe@unlish.com>
parents:
10076
diff
changeset
|
722 |
self.info("sql: %r\n args: %s\ndbms message: %r", |
4682
4994901b7379
don't issue critical message when trying to alter sqlite db during test
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4517
diff
changeset
|
723 |
query, args, ex.args[0]) |
2618
ff9b0d5bd884
[F repo sqlite schema changes] don't rollback on potentially expected schema changes failure
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2611
diff
changeset
|
724 |
if rollback: |
ff9b0d5bd884
[F repo sqlite schema changes] don't rollback on potentially expected schema changes failure
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2611
diff
changeset
|
725 |
try: |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
726 |
cnx.cnxset.rollback() |
4692
11a040e2601c
[test] also hide this message during test
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4682
diff
changeset
|
727 |
if self.repo.config.mode != 'test': |
10178
4b33246cb43c
[native] Change SQL errors log level.
Christophe de Vienne <christophe@unlish.com>
parents:
10076
diff
changeset
|
728 |
self.debug('transaction has been rolled back') |
8695
358d8bed9626
[toward-py3k] rewrite to "except AnException as exc:" (part of #2711624)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
8694
diff
changeset
|
729 |
except Exception as ex: |
2618
ff9b0d5bd884
[F repo sqlite schema changes] don't rollback on potentially expected schema changes failure
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2611
diff
changeset
|
730 |
pass |
6211
e9d125fd1465
nicer error reporting for unique together constraints
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6201
diff
changeset
|
731 |
if ex.__class__.__name__ == 'IntegrityError': |
e9d125fd1465
nicer error reporting for unique together constraints
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6201
diff
changeset
|
732 |
# need string comparison because of various backends |
e9d125fd1465
nicer error reporting for unique together constraints
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6201
diff
changeset
|
733 |
for arg in ex.args: |
9375
8e88576787c3
[schema] fix unique together index handling
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9336
diff
changeset
|
734 |
# postgres, sqlserver |
8e88576787c3
[schema] fix unique together index handling
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9336
diff
changeset
|
735 |
mo = re.search("unique_[a-z0-9]{32}", arg) |
6211
e9d125fd1465
nicer error reporting for unique together constraints
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6201
diff
changeset
|
736 |
if mo is not None: |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
737 |
raise UniqueTogetherError(cnx, cstrname=mo.group(0)) |
10053
079b5279a2cd
[server] Handle unique constraint violations under recent sqlite
Julien Cristau <julien.cristau@logilab.fr>
parents:
10052
diff
changeset
|
738 |
# old sqlite |
10329
f92823a66f18
[sqlite] Fix integrity error parsing
Christophe de Vienne <christophe@unlish.com>
parents:
10272
diff
changeset
|
739 |
mo = re.search('columns? (.*) (?:is|are) not unique', arg) |
6247
f7cb092d2296
unique_together: recast exception raised by sqlite
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6211
diff
changeset
|
740 |
if mo is not None: # sqlite in use |
8820
dbffb6959564
server/source/native: fix wrong usage of .lstrip that produce garbled error messages (closes #2777641)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8546
diff
changeset
|
741 |
# we left chop the 'cw_' prefix of attribute names |
dbffb6959564
server/source/native: fix wrong usage of .lstrip that produce garbled error messages (closes #2777641)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8546
diff
changeset
|
742 |
rtypes = [c.strip()[3:] |
dbffb6959564
server/source/native: fix wrong usage of .lstrip that produce garbled error messages (closes #2777641)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8546
diff
changeset
|
743 |
for c in mo.group(1).split(',')] |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
744 |
raise UniqueTogetherError(cnx, rtypes=rtypes) |
10053
079b5279a2cd
[server] Handle unique constraint violations under recent sqlite
Julien Cristau <julien.cristau@logilab.fr>
parents:
10052
diff
changeset
|
745 |
# sqlite after http://www.sqlite.org/cgi/src/info/c80e229dd9c1230a |
079b5279a2cd
[server] Handle unique constraint violations under recent sqlite
Julien Cristau <julien.cristau@logilab.fr>
parents:
10052
diff
changeset
|
746 |
if arg.startswith('UNIQUE constraint failed:'): |
079b5279a2cd
[server] Handle unique constraint violations under recent sqlite
Julien Cristau <julien.cristau@logilab.fr>
parents:
10052
diff
changeset
|
747 |
# message looks like: "UNIQUE constraint failed: foo.cw_bar, foo.cw_baz" |
079b5279a2cd
[server] Handle unique constraint violations under recent sqlite
Julien Cristau <julien.cristau@logilab.fr>
parents:
10052
diff
changeset
|
748 |
# so drop the prefix, split on comma, drop the tablenames, and drop "cw_" |
079b5279a2cd
[server] Handle unique constraint violations under recent sqlite
Julien Cristau <julien.cristau@logilab.fr>
parents:
10052
diff
changeset
|
749 |
columns = arg.split(':', 1)[1].split(',') |
079b5279a2cd
[server] Handle unique constraint violations under recent sqlite
Julien Cristau <julien.cristau@logilab.fr>
parents:
10052
diff
changeset
|
750 |
rtypes = [c.split('.', 1)[1].strip()[3:] for c in columns] |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
751 |
raise UniqueTogetherError(cnx, rtypes=rtypes) |
10446
1e6655cff5ab
add IUserFriendlyError adapter for violation of check constraints
Julien Cristau <julien.cristau@logilab.fr>
parents:
10411
diff
changeset
|
752 |
|
1e6655cff5ab
add IUserFriendlyError adapter for violation of check constraints
Julien Cristau <julien.cristau@logilab.fr>
parents:
10411
diff
changeset
|
753 |
mo = re.search('"cstr[a-f0-9]{32}"', arg) |
1e6655cff5ab
add IUserFriendlyError adapter for violation of check constraints
Julien Cristau <julien.cristau@logilab.fr>
parents:
10411
diff
changeset
|
754 |
if mo is not None: |
1e6655cff5ab
add IUserFriendlyError adapter for violation of check constraints
Julien Cristau <julien.cristau@logilab.fr>
parents:
10411
diff
changeset
|
755 |
# postgresql |
1e6655cff5ab
add IUserFriendlyError adapter for violation of check constraints
Julien Cristau <julien.cristau@logilab.fr>
parents:
10411
diff
changeset
|
756 |
raise ViolatedConstraint(cnx, cstrname=mo.group(0)[1:-1]) |
1e6655cff5ab
add IUserFriendlyError adapter for violation of check constraints
Julien Cristau <julien.cristau@logilab.fr>
parents:
10411
diff
changeset
|
757 |
if arg.startswith('CHECK constraint failed:'): |
1e6655cff5ab
add IUserFriendlyError adapter for violation of check constraints
Julien Cristau <julien.cristau@logilab.fr>
parents:
10411
diff
changeset
|
758 |
# sqlite3 (new) |
1e6655cff5ab
add IUserFriendlyError adapter for violation of check constraints
Julien Cristau <julien.cristau@logilab.fr>
parents:
10411
diff
changeset
|
759 |
raise ViolatedConstraint(cnx, cstrname=arg.split(':', 1)[1].strip()) |
1e6655cff5ab
add IUserFriendlyError adapter for violation of check constraints
Julien Cristau <julien.cristau@logilab.fr>
parents:
10411
diff
changeset
|
760 |
mo = re.match('^constraint (cstr.*) failed$', arg) |
1e6655cff5ab
add IUserFriendlyError adapter for violation of check constraints
Julien Cristau <julien.cristau@logilab.fr>
parents:
10411
diff
changeset
|
761 |
if mo is not None: |
1e6655cff5ab
add IUserFriendlyError adapter for violation of check constraints
Julien Cristau <julien.cristau@logilab.fr>
parents:
10411
diff
changeset
|
762 |
# sqlite3 (old) |
1e6655cff5ab
add IUserFriendlyError adapter for violation of check constraints
Julien Cristau <julien.cristau@logilab.fr>
parents:
10411
diff
changeset
|
763 |
raise ViolatedConstraint(cnx, cstrname=mo.group(1)) |
0 | 764 |
raise |
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
765 |
return cursor |
1792 | 766 |
|
10477
ee21c559f94f
implement a simple statsd logger (closes #5488711)
David Douard <david.douard@logilab.fr>
parents:
10473
diff
changeset
|
767 |
@statsd_timeit |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
768 |
def doexecmany(self, cnx, query, args): |
0 | 769 |
"""Execute a query. |
770 |
it's a function just so that it shows up in profiling |
|
771 |
""" |
|
2593
16d9419a4a79
F: start to handle binary debug log level on the server side
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2589
diff
changeset
|
772 |
if server.DEBUG & server.DBG_SQL: |
10589
7c23b7de2b8d
[py3k] print function
Samuel Trégouët <samuel.tregouet@logilab.fr>
parents:
10580
diff
changeset
|
773 |
print('execmany', query, 'with', len(args), 'arguments', cnx.cnxset.cnx) |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
774 |
cursor = cnx.cnxset.cu |
0 | 775 |
try: |
9701
46c8d8701240
an unicode string -> a unicode string
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9492
diff
changeset
|
776 |
# str(query) to avoid error if it's a unicode string |
0 | 777 |
cursor.executemany(str(query), args) |
8695
358d8bed9626
[toward-py3k] rewrite to "except AnException as exc:" (part of #2711624)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
8694
diff
changeset
|
778 |
except Exception as ex: |
4682
4994901b7379
don't issue critical message when trying to alter sqlite db during test
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4517
diff
changeset
|
779 |
if self.repo.config.mode != 'test': |
4994901b7379
don't issue critical message when trying to alter sqlite db during test
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4517
diff
changeset
|
780 |
# during test we get those message when trying to alter sqlite |
4994901b7379
don't issue critical message when trying to alter sqlite db during test
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4517
diff
changeset
|
781 |
# db schema |
4994901b7379
don't issue critical message when trying to alter sqlite db during test
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4517
diff
changeset
|
782 |
self.critical("sql many: %r\n args: %s\ndbms message: %r", |
4994901b7379
don't issue critical message when trying to alter sqlite db during test
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4517
diff
changeset
|
783 |
query, args, ex.args[0]) |
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
784 |
try: |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
785 |
cnx.cnxset.rollback() |
4692
11a040e2601c
[test] also hide this message during test
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4682
diff
changeset
|
786 |
if self.repo.config.mode != 'test': |
9267
24d9b86dfa54
spelling: rollbacked -> rolled back
Rémi Cardona <remi.cardona@logilab.fr>
parents:
9131
diff
changeset
|
787 |
self.critical('transaction has been rolled back') |
7645
cb2459aaa7d7
[source] closes #1816412: try to reconnect to dbms if connection is reset in eid_type_source()
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
7562
diff
changeset
|
788 |
except Exception: |
2306
95da5d9f0870
give session to doexec so it's able to rollback the connection on unexpected error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
2072
diff
changeset
|
789 |
pass |
0 | 790 |
raise |
1792 | 791 |
|
0 | 792 |
# short cut to method requiring advanced db helper usage ################## |
1792 | 793 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
794 |
def update_rdef_column(self, cnx, rdef): |
5891
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
795 |
"""update physical column for a relation definition (final or inlined) |
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
796 |
""" |
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
797 |
table, column = rdef_table_column(rdef) |
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
798 |
coltype, allownull = rdef_physical_info(self.dbhelper, rdef) |
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
799 |
if not self.dbhelper.alter_column_support: |
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
800 |
self.error("backend can't alter %s.%s to %s%s", table, column, coltype, |
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
801 |
not allownull and 'NOT NULL' or '') |
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
802 |
return |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
803 |
self.dbhelper.change_col_type(LogCursor(cnx.cnxset.cu), |
5891
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
804 |
table, column, coltype, allownull) |
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
805 |
self.info('altered %s.%s: now %s%s', table, column, coltype, |
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
806 |
not allownull and 'NOT NULL' or '') |
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
807 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
808 |
def update_rdef_null_allowed(self, cnx, rdef): |
5891
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
809 |
"""update NULL / NOT NULL of physical column for a relation definition |
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
810 |
(final or inlined) |
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
811 |
""" |
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
812 |
if not self.dbhelper.alter_column_support: |
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
813 |
# not supported (and NOT NULL not set by yams in that case, so no |
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
814 |
# worry) |
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
815 |
return |
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
816 |
table, column = rdef_table_column(rdef) |
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
817 |
coltype, allownull = rdef_physical_info(self.dbhelper, rdef) |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
818 |
self.dbhelper.set_null_allowed(LogCursor(cnx.cnxset.cu), |
5891
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
819 |
table, column, coltype, allownull) |
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
820 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
821 |
def update_rdef_indexed(self, cnx, rdef): |
5891
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
822 |
table, column = rdef_table_column(rdef) |
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
823 |
if rdef.indexed: |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
824 |
self.create_index(cnx, table, column) |
5891
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
825 |
else: |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
826 |
self.drop_index(cnx, table, column) |
5891
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
827 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
828 |
def update_rdef_unique(self, cnx, rdef): |
5891
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
829 |
table, column = rdef_table_column(rdef) |
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
830 |
if rdef.constraint_by_type('UniqueConstraint'): |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
831 |
self.create_index(cnx, table, column, unique=True) |
5891
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
832 |
else: |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
833 |
self.drop_index(cnx, table, column, unique=True) |
5891
99024ad59223
[schema migration] import refactoring to fix #1109558 and enhances things on the way
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5849
diff
changeset
|
834 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
835 |
def create_index(self, cnx, table, column, unique=False): |
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
836 |
cursor = LogCursor(cnx.cnxset.cu) |
0 | 837 |
self.dbhelper.create_index(cursor, table, column, unique) |
1792 | 838 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
839 |
def drop_index(self, cnx, table, column, unique=False): |
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
840 |
cursor = LogCursor(cnx.cnxset.cu) |
0 | 841 |
self.dbhelper.drop_index(cursor, table, column, unique) |
842 |
||
843 |
# system source interface ################################################# |
|
844 |
||
10366
38c7598b5c61
[connection] remove the `mode` attribute
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10365
diff
changeset
|
845 |
def _eid_type_source(self, cnx, eid, sql): |
0 | 846 |
try: |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
847 |
res = self.doexec(cnx, sql).fetchone() |
7646
91a984ba420f
[native source] fix bug introduced in 7645:cb2459aaa7d7: fetchone() may return None
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7645
diff
changeset
|
848 |
if res is not None: |
91a984ba420f
[native source] fix bug introduced in 7645:cb2459aaa7d7: fetchone() may return None
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7645
diff
changeset
|
849 |
return res |
7645
cb2459aaa7d7
[source] closes #1816412: try to reconnect to dbms if connection is reset in eid_type_source()
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
7562
diff
changeset
|
850 |
except Exception: |
cb2459aaa7d7
[source] closes #1816412: try to reconnect to dbms if connection is reset in eid_type_source()
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
7562
diff
changeset
|
851 |
self.exception('failed to query entities table for eid %s', eid) |
cb2459aaa7d7
[source] closes #1816412: try to reconnect to dbms if connection is reset in eid_type_source()
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
7562
diff
changeset
|
852 |
raise UnknownEid(eid) |
cb2459aaa7d7
[source] closes #1816412: try to reconnect to dbms if connection is reset in eid_type_source()
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
7562
diff
changeset
|
853 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
854 |
def eid_type_source(self, cnx, eid): # pylint: disable=E0202 |
10575
036f5964e6fe
[server/sources] fix docstring for eid_type_source
Julien Cristau <julien.cristau@logilab.fr>
parents:
10550
diff
changeset
|
855 |
"""return a tuple (type, extid, source) for the entity with id <eid>""" |
9469
032825bbacab
[multi-sources-removal] Drop entities.source column
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9468
diff
changeset
|
856 |
sql = 'SELECT type, extid, asource FROM entities WHERE eid=%s' % eid |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
857 |
res = self._eid_type_source(cnx, eid, sql) |
10580
5fbdbbe51867
[server/sources] add decode_extid method
Julien Cristau <julien.cristau@logilab.fr>
parents:
10550
diff
changeset
|
858 |
if not isinstance(res, list): |
5fbdbbe51867
[server/sources] add decode_extid method
Julien Cristau <julien.cristau@logilab.fr>
parents:
10550
diff
changeset
|
859 |
res = list(res) |
5fbdbbe51867
[server/sources] add decode_extid method
Julien Cristau <julien.cristau@logilab.fr>
parents:
10550
diff
changeset
|
860 |
res[-2] = self.decode_extid(res[-2]) |
7543
570522300e22
[ms, entity metas] add 'actual source' to entities table / base entity metadata cache. Closes #1767090
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7536
diff
changeset
|
861 |
return res |
570522300e22
[ms, entity metas] add 'actual source' to entities table / base entity metadata cache. Closes #1767090
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7536
diff
changeset
|
862 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
863 |
def eid_type_source_pre_131(self, cnx, eid): |
10575
036f5964e6fe
[server/sources] fix docstring for eid_type_source
Julien Cristau <julien.cristau@logilab.fr>
parents:
10550
diff
changeset
|
864 |
"""return a tuple (type, extid, source) for the entity with id <eid>""" |
9469
032825bbacab
[multi-sources-removal] Drop entities.source column
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9468
diff
changeset
|
865 |
sql = 'SELECT type, extid FROM entities WHERE eid=%s' % eid |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
866 |
res = self._eid_type_source(cnx, eid, sql) |
7543
570522300e22
[ms, entity metas] add 'actual source' to entities table / base entity metadata cache. Closes #1767090
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7536
diff
changeset
|
867 |
if not isinstance(res, list): |
570522300e22
[ms, entity metas] add 'actual source' to entities table / base entity metadata cache. Closes #1767090
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7536
diff
changeset
|
868 |
res = list(res) |
10580
5fbdbbe51867
[server/sources] add decode_extid method
Julien Cristau <julien.cristau@logilab.fr>
parents:
10550
diff
changeset
|
869 |
res[-1] = self.decode_extid(extid) |
10270
1ae64186af94
[migration] reorder system tables / meta-schema migration into bootstrap_migration
Erica Marco <erica.marco@logilab.fr>
parents:
10197
diff
changeset
|
870 |
res.append("system") |
0 | 871 |
return res |
872 |
||
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
873 |
def extid2eid(self, cnx, extid): |
1952
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
874 |
"""get eid from an external id. Return None if no record found.""" |
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
875 |
assert isinstance(extid, str) |
10209
4c64a41c0a1d
Use a moved_entities table to record external entities moved to the system source
Julien Cristau <julien.cristau@logilab.fr>
parents:
10208
diff
changeset
|
876 |
args = {'x': b64encode(extid)} |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
877 |
cursor = self.doexec(cnx, |
9469
032825bbacab
[multi-sources-removal] Drop entities.source column
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9468
diff
changeset
|
878 |
'SELECT eid FROM entities WHERE extid=%(x)s', |
10209
4c64a41c0a1d
Use a moved_entities table to record external entities moved to the system source
Julien Cristau <julien.cristau@logilab.fr>
parents:
10208
diff
changeset
|
879 |
args) |
0 | 880 |
# XXX testing rowcount cause strange bug with sqlite, results are there |
881 |
# but rowcount is 0 |
|
1792 | 882 |
#if cursor.rowcount > 0: |
0 | 883 |
try: |
884 |
result = cursor.fetchone() |
|
885 |
if result: |
|
1954 | 886 |
return result[0] |
7645
cb2459aaa7d7
[source] closes #1816412: try to reconnect to dbms if connection is reset in eid_type_source()
Adrien Di Mascio <Adrien.DiMascio@logilab.fr>
parents:
7562
diff
changeset
|
887 |
except Exception: |
0 | 888 |
pass |
10209
4c64a41c0a1d
Use a moved_entities table to record external entities moved to the system source
Julien Cristau <julien.cristau@logilab.fr>
parents:
10208
diff
changeset
|
889 |
cursor = self.doexec(cnx, |
4c64a41c0a1d
Use a moved_entities table to record external entities moved to the system source
Julien Cristau <julien.cristau@logilab.fr>
parents:
10208
diff
changeset
|
890 |
'SELECT eid FROM moved_entities WHERE extid=%(x)s', |
4c64a41c0a1d
Use a moved_entities table to record external entities moved to the system source
Julien Cristau <julien.cristau@logilab.fr>
parents:
10208
diff
changeset
|
891 |
args) |
4c64a41c0a1d
Use a moved_entities table to record external entities moved to the system source
Julien Cristau <julien.cristau@logilab.fr>
parents:
10208
diff
changeset
|
892 |
try: |
4c64a41c0a1d
Use a moved_entities table to record external entities moved to the system source
Julien Cristau <julien.cristau@logilab.fr>
parents:
10208
diff
changeset
|
893 |
result = cursor.fetchone() |
4c64a41c0a1d
Use a moved_entities table to record external entities moved to the system source
Julien Cristau <julien.cristau@logilab.fr>
parents:
10208
diff
changeset
|
894 |
if result: |
4c64a41c0a1d
Use a moved_entities table to record external entities moved to the system source
Julien Cristau <julien.cristau@logilab.fr>
parents:
10208
diff
changeset
|
895 |
# entity was moved to the system source, return negative |
4c64a41c0a1d
Use a moved_entities table to record external entities moved to the system source
Julien Cristau <julien.cristau@logilab.fr>
parents:
10208
diff
changeset
|
896 |
# number to tell the external source to ignore it |
4c64a41c0a1d
Use a moved_entities table to record external entities moved to the system source
Julien Cristau <julien.cristau@logilab.fr>
parents:
10208
diff
changeset
|
897 |
return -result[0] |
4c64a41c0a1d
Use a moved_entities table to record external entities moved to the system source
Julien Cristau <julien.cristau@logilab.fr>
parents:
10208
diff
changeset
|
898 |
except Exception: |
4c64a41c0a1d
Use a moved_entities table to record external entities moved to the system source
Julien Cristau <julien.cristau@logilab.fr>
parents:
10208
diff
changeset
|
899 |
pass |
0 | 900 |
return None |
1792 | 901 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
902 |
def _handle_is_relation_sql(self, cnx, sql, attrs): |
8625
7ee0752178e5
[dataimport] Add SQL Store for faster import - works ONLY with Postgres for now, as it requires "copy from" command - closes #2410822
Vincent Michel <vincent.michel@logilab.fr>
parents:
8573
diff
changeset
|
903 |
""" Handler for specific is_relation sql that may be |
7ee0752178e5
[dataimport] Add SQL Store for faster import - works ONLY with Postgres for now, as it requires "copy from" command - closes #2410822
Vincent Michel <vincent.michel@logilab.fr>
parents:
8573
diff
changeset
|
904 |
overwritten in some stores""" |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
905 |
self.doexec(cnx, sql % attrs) |
8625
7ee0752178e5
[dataimport] Add SQL Store for faster import - works ONLY with Postgres for now, as it requires "copy from" command - closes #2410822
Vincent Michel <vincent.michel@logilab.fr>
parents:
8573
diff
changeset
|
906 |
|
7ee0752178e5
[dataimport] Add SQL Store for faster import - works ONLY with Postgres for now, as it requires "copy from" command - closes #2410822
Vincent Michel <vincent.michel@logilab.fr>
parents:
8573
diff
changeset
|
907 |
_handle_insert_entity_sql = doexec |
7ee0752178e5
[dataimport] Add SQL Store for faster import - works ONLY with Postgres for now, as it requires "copy from" command - closes #2410822
Vincent Michel <vincent.michel@logilab.fr>
parents:
8573
diff
changeset
|
908 |
_handle_is_instance_of_sql = _handle_source_relation_sql = _handle_is_relation_sql |
7ee0752178e5
[dataimport] Add SQL Store for faster import - works ONLY with Postgres for now, as it requires "copy from" command - closes #2410822
Vincent Michel <vincent.michel@logilab.fr>
parents:
8573
diff
changeset
|
909 |
|
9664
5ef5494b6b0b
[repository] complete argument is no more needed since we have no more 'true' multisource.
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9611
diff
changeset
|
910 |
def add_info(self, cnx, entity, source, extid): |
0 | 911 |
"""add type and source info for an eid into the system table""" |
9802
9f815b1f2516
[repository,tests] use the new connection api (part 1/2)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9770
diff
changeset
|
912 |
assert cnx.cnxset is not None |
0 | 913 |
# begin by inserting eid/type/source/extid into the entities table |
1952
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
914 |
if extid is not None: |
10676
9d6b79081bad
[py3k] use text/binary types from six
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10662
diff
changeset
|
915 |
assert isinstance(extid, binary_type) |
1952
8e19c813750d
fix extid handling: ensure encoded string is given, and store them as base64 (see note in native.py).
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
1792
diff
changeset
|
916 |
extid = b64encode(extid) |
10676
9d6b79081bad
[py3k] use text/binary types from six
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10662
diff
changeset
|
917 |
attrs = {'type': text_type(entity.cw_etype), 'eid': entity.eid, 'extid': extid and text_type(extid), |
9d6b79081bad
[py3k] use text/binary types from six
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10662
diff
changeset
|
918 |
'asource': text_type(source.uri)} |
9802
9f815b1f2516
[repository,tests] use the new connection api (part 1/2)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9770
diff
changeset
|
919 |
self._handle_insert_entity_sql(cnx, self.sqlgen.insert('entities', attrs), attrs) |
6427
c8a5ac2d1eaa
[schema / sources] store data sources as cubicweb entities
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6426
diff
changeset
|
920 |
# insert core relations: is, is_instance_of and cw_source |
6851
824d5b6eae7f
[repo] kill no more needed 'recreate' feature
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6849
diff
changeset
|
921 |
try: |
9802
9f815b1f2516
[repository,tests] use the new connection api (part 1/2)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9770
diff
changeset
|
922 |
self._handle_is_relation_sql(cnx, 'INSERT INTO is_relation(eid_from,eid_to) VALUES (%s,%s)', |
9f815b1f2516
[repository,tests] use the new connection api (part 1/2)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9770
diff
changeset
|
923 |
(entity.eid, eschema_eid(cnx, entity.e_schema))) |
6851
824d5b6eae7f
[repo] kill no more needed 'recreate' feature
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6849
diff
changeset
|
924 |
except IndexError: |
824d5b6eae7f
[repo] kill no more needed 'recreate' feature
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6849
diff
changeset
|
925 |
# during schema serialization, skip |
824d5b6eae7f
[repo] kill no more needed 'recreate' feature
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6849
diff
changeset
|
926 |
pass |
824d5b6eae7f
[repo] kill no more needed 'recreate' feature
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6849
diff
changeset
|
927 |
else: |
824d5b6eae7f
[repo] kill no more needed 'recreate' feature
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6849
diff
changeset
|
928 |
for eschema in entity.e_schema.ancestors() + [entity.e_schema]: |
9802
9f815b1f2516
[repository,tests] use the new connection api (part 1/2)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9770
diff
changeset
|
929 |
self._handle_is_relation_sql(cnx, |
8625
7ee0752178e5
[dataimport] Add SQL Store for faster import - works ONLY with Postgres for now, as it requires "copy from" command - closes #2410822
Vincent Michel <vincent.michel@logilab.fr>
parents:
8573
diff
changeset
|
930 |
'INSERT INTO is_instance_of_relation(eid_from,eid_to) VALUES (%s,%s)', |
9802
9f815b1f2516
[repository,tests] use the new connection api (part 1/2)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9770
diff
changeset
|
931 |
(entity.eid, eschema_eid(cnx, eschema))) |
6851
824d5b6eae7f
[repo] kill no more needed 'recreate' feature
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6849
diff
changeset
|
932 |
if 'CWSource' in self.schema and source.eid is not None: # else, cw < 3.10 |
9802
9f815b1f2516
[repository,tests] use the new connection api (part 1/2)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9770
diff
changeset
|
933 |
self._handle_is_relation_sql(cnx, 'INSERT INTO cw_source_relation(eid_from,eid_to) VALUES (%s,%s)', |
8625
7ee0752178e5
[dataimport] Add SQL Store for faster import - works ONLY with Postgres for now, as it requires "copy from" command - closes #2410822
Vincent Michel <vincent.michel@logilab.fr>
parents:
8573
diff
changeset
|
934 |
(entity.eid, source.eid)) |
4806
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
935 |
# now we can update the full text index |
10631
1ab79c435fae
consider .do_fti flag in index_entity method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10329
diff
changeset
|
936 |
if self.need_fti_indexation(entity.cw_etype): |
9802
9f815b1f2516
[repository,tests] use the new connection api (part 1/2)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9770
diff
changeset
|
937 |
self.index_entity(cnx, entity=entity) |
4806
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
938 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
939 |
def update_info(self, cnx, entity, need_fti_update): |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
940 |
"""mark entity as being modified, fulltext reindex if needed""" |
10631
1ab79c435fae
consider .do_fti flag in index_entity method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10329
diff
changeset
|
941 |
if need_fti_update: |
4806
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
942 |
# reindex the entity only if this query is updating at least |
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
943 |
# one indexable attribute |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
944 |
self.index_entity(cnx, entity=entity) |
0 | 945 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
946 |
def delete_info_multi(self, cnx, entities): |
7501
2983dd24494a
[repository] refactor/cleanup entity deletion methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7342
diff
changeset
|
947 |
"""delete system information on deletion of a list of entities with the |
2983dd24494a
[repository] refactor/cleanup entity deletion methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7342
diff
changeset
|
948 |
same etype and belinging to the same source |
2983dd24494a
[repository] refactor/cleanup entity deletion methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7342
diff
changeset
|
949 |
|
5067
adc2122eed03
[repo] more efficient eid cache operations handling based on set_operation; refactor
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5061
diff
changeset
|
950 |
* update the fti |
7501
2983dd24494a
[repository] refactor/cleanup entity deletion methods
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7342
diff
changeset
|
951 |
* remove record from the `entities` table |
6889
37668bf302f5
improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6850
diff
changeset
|
952 |
""" |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
953 |
self.fti_unindex_entities(cnx, entities) |
6889
37668bf302f5
improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6850
diff
changeset
|
954 |
attrs = {'eid': '(%s)' % ','.join([str(_e.eid) for _e in entities])} |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
955 |
self.doexec(cnx, self.sqlgen.delete_many('entities', attrs), attrs) |
0 | 956 |
|
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
957 |
# undo support ############################################################# |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
958 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
959 |
def undoable_transactions(self, cnx, ueid=None, **actionfilters): |
10356
a009a31fb1ea
[connection] eliminate ClientConnection leftovers
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10351
diff
changeset
|
960 |
"""See :class:`cubicweb.repoapi.Connection.undoable_transactions`""" |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
961 |
# force filtering to connection's user if not a manager |
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
962 |
if not cnx.user.is_in_group('managers'): |
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
963 |
ueid = cnx.user.eid |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
964 |
restr = {} |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
965 |
if ueid is not None: |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
966 |
restr['tx_user'] = ueid |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
967 |
sql = self.sqlgen.select('transactions', restr, ('tx_uuid', 'tx_time', 'tx_user')) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
968 |
if actionfilters: |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
969 |
# we will need subqueries to filter transactions according to |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
970 |
# actions done |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
971 |
tearestr = {} # filters on the tx_entity_actions table |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
972 |
trarestr = {} # filters on the tx_relation_actions table |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
973 |
genrestr = {} # generic filters, appliyable to both table |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
974 |
# unless public explicitly set to false, we only consider public |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
975 |
# actions |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
976 |
if actionfilters.pop('public', True): |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
977 |
genrestr['txa_public'] = True |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
978 |
# put additional filters in trarestr and/or tearestr |
10662
10942ed172de
[py3k] dict.iteritems → dict.items
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10651
diff
changeset
|
979 |
for key, val in actionfilters.items(): |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
980 |
if key == 'etype': |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
981 |
# filtering on etype implies filtering on entity actions |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
982 |
# only, and with no eid specified |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
983 |
assert actionfilters.get('action', 'C') in 'CUD' |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
984 |
assert not 'eid' in actionfilters |
10676
9d6b79081bad
[py3k] use text/binary types from six
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10662
diff
changeset
|
985 |
tearestr['etype'] = text_type(val) |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
986 |
elif key == 'eid': |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
987 |
# eid filter may apply to 'eid' of tx_entity_actions or to |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
988 |
# 'eid_from' OR 'eid_to' of tx_relation_actions |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
989 |
if actionfilters.get('action', 'C') in 'CUD': |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
990 |
tearestr['eid'] = val |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
991 |
if actionfilters.get('action', 'A') in 'AR': |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
992 |
trarestr['eid_from'] = val |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
993 |
trarestr['eid_to'] = val |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
994 |
elif key == 'action': |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
995 |
if val in 'CUD': |
10676
9d6b79081bad
[py3k] use text/binary types from six
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10662
diff
changeset
|
996 |
tearestr['txa_action'] = text_type(val) |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
997 |
else: |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
998 |
assert val in 'AR' |
10676
9d6b79081bad
[py3k] use text/binary types from six
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10662
diff
changeset
|
999 |
trarestr['txa_action'] = text_type(val) |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1000 |
else: |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1001 |
raise AssertionError('unknow filter %s' % key) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1002 |
assert trarestr or tearestr, "can't only filter on 'public'" |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1003 |
subqsqls = [] |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1004 |
# append subqueries to the original query, using EXISTS() |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1005 |
if trarestr or (genrestr and not tearestr): |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1006 |
trarestr.update(genrestr) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1007 |
trasql = self.sqlgen.select('tx_relation_actions', trarestr, ('1',)) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1008 |
if 'eid_from' in trarestr: |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1009 |
# replace AND by OR between eid_from/eid_to restriction |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1010 |
trasql = sql_or_clauses(trasql, ['eid_from = %(eid_from)s', |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1011 |
'eid_to = %(eid_to)s']) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1012 |
trasql += ' AND transactions.tx_uuid=tx_relation_actions.tx_uuid' |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1013 |
subqsqls.append('EXISTS(%s)' % trasql) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1014 |
if tearestr or (genrestr and not trarestr): |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1015 |
tearestr.update(genrestr) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1016 |
teasql = self.sqlgen.select('tx_entity_actions', tearestr, ('1',)) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1017 |
teasql += ' AND transactions.tx_uuid=tx_entity_actions.tx_uuid' |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1018 |
subqsqls.append('EXISTS(%s)' % teasql) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1019 |
if restr: |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1020 |
sql += ' AND %s' % ' OR '.join(subqsqls) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1021 |
else: |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1022 |
sql += ' WHERE %s' % ' OR '.join(subqsqls) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1023 |
restr.update(trarestr) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1024 |
restr.update(tearestr) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1025 |
# we want results ordered by transaction's time descendant |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1026 |
sql += ' ORDER BY tx_time DESC' |
10365
21461f80f348
[connection] remove ensure_cnx_set context manager uses
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10364
diff
changeset
|
1027 |
cu = self.doexec(cnx, sql, restr) |
21461f80f348
[connection] remove ensure_cnx_set context manager uses
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10364
diff
changeset
|
1028 |
# turn results into transaction objects |
21461f80f348
[connection] remove ensure_cnx_set context manager uses
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10364
diff
changeset
|
1029 |
return [tx.Transaction(cnx, *args) for args in cu.fetchall()] |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1030 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1031 |
def tx_info(self, cnx, txuuid): |
10356
a009a31fb1ea
[connection] eliminate ClientConnection leftovers
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10351
diff
changeset
|
1032 |
"""See :class:`cubicweb.repoapi.Connection.transaction_info`""" |
10676
9d6b79081bad
[py3k] use text/binary types from six
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10662
diff
changeset
|
1033 |
return tx.Transaction(cnx, txuuid, *self._tx_info(cnx, text_type(txuuid))) |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1034 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1035 |
def tx_actions(self, cnx, txuuid, public): |
10356
a009a31fb1ea
[connection] eliminate ClientConnection leftovers
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10351
diff
changeset
|
1036 |
"""See :class:`cubicweb.repoapi.Connection.transaction_actions`""" |
10676
9d6b79081bad
[py3k] use text/binary types from six
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10662
diff
changeset
|
1037 |
txuuid = text_type(txuuid) |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1038 |
self._tx_info(cnx, txuuid) |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1039 |
restr = {'tx_uuid': txuuid} |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1040 |
if public: |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1041 |
restr['txa_public'] = True |
5072
072ae171aeb0
[cleanup] style fixes, add nodes, 0.2 cents refactorings
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5071
diff
changeset
|
1042 |
# XXX use generator to avoid loading everything in memory? |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1043 |
sql = self.sqlgen.select('tx_entity_actions', restr, |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1044 |
('txa_action', 'txa_public', 'txa_order', |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1045 |
'etype', 'eid', 'changes')) |
9996
65bb21409292
[server] hold connection to the db in tx_actions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9984
diff
changeset
|
1046 |
with cnx.ensure_cnx_set: |
65bb21409292
[server] hold connection to the db in tx_actions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9984
diff
changeset
|
1047 |
cu = self.doexec(cnx, sql, restr) |
10602
4845012cfc8e
[py3k] import 'pickle' using six.moves
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10589
diff
changeset
|
1048 |
actions = [tx.EntityAction(a,p,o,et,e,c and pickle.loads(self.binary_to_str(c))) |
9996
65bb21409292
[server] hold connection to the db in tx_actions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9984
diff
changeset
|
1049 |
for a,p,o,et,e,c in cu.fetchall()] |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1050 |
sql = self.sqlgen.select('tx_relation_actions', restr, |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1051 |
('txa_action', 'txa_public', 'txa_order', |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1052 |
'rtype', 'eid_from', 'eid_to')) |
9996
65bb21409292
[server] hold connection to the db in tx_actions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9984
diff
changeset
|
1053 |
with cnx.ensure_cnx_set: |
65bb21409292
[server] hold connection to the db in tx_actions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9984
diff
changeset
|
1054 |
cu = self.doexec(cnx, sql, restr) |
65bb21409292
[server] hold connection to the db in tx_actions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9984
diff
changeset
|
1055 |
actions += [tx.RelationAction(*args) for args in cu.fetchall()] |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1056 |
return sorted(actions, key=lambda x: x.order) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1057 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1058 |
def undo_transaction(self, cnx, txuuid): |
10356
a009a31fb1ea
[connection] eliminate ClientConnection leftovers
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10351
diff
changeset
|
1059 |
"""See :class:`cubicweb.repoapi.Connection.undo_transaction` |
5071
8631bb9f6e73
[undo] during undoing, call hooks in the [active]integrity and undo categories
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5070
diff
changeset
|
1060 |
|
8631bb9f6e73
[undo] during undoing, call hooks in the [active]integrity and undo categories
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5070
diff
changeset
|
1061 |
important note: while undoing of a transaction, only hooks in the |
8631bb9f6e73
[undo] during undoing, call hooks in the [active]integrity and undo categories
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5070
diff
changeset
|
1062 |
'integrity', 'activeintegrity' and 'undo' categories are called. |
8631bb9f6e73
[undo] during undoing, call hooks in the [active]integrity and undo categories
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5070
diff
changeset
|
1063 |
""" |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1064 |
errors = [] |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1065 |
cnx.transaction_data['undoing_uuid'] = txuuid |
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1066 |
with cnx.deny_all_hooks_but('integrity', 'activeintegrity', 'undo'): |
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1067 |
with cnx.security_enabled(read=False): |
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1068 |
for action in reversed(self.tx_actions(cnx, txuuid, False)): |
4943
7f5b83578fec
disable security when undoing
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4913
diff
changeset
|
1069 |
undomethod = getattr(self, '_undo_%s' % action.action.lower()) |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1070 |
errors += undomethod(cnx, action) |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1071 |
# remove the transactions record |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1072 |
self.doexec(cnx, |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1073 |
"DELETE FROM transactions WHERE tx_uuid='%s'" % txuuid) |
8265
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1074 |
if errors: |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1075 |
raise UndoTransactionException(txuuid, errors) |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1076 |
else: |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1077 |
return |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1078 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1079 |
def start_undoable_transaction(self, cnx, uuid): |
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1080 |
"""connection callback to insert a transaction record in the transactions |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1081 |
table when some undoable transaction is started |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1082 |
""" |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1083 |
ueid = cnx.user.eid |
7922
d307c3817782
[repository] use utcnow instead of now for some internal timetamps (closes #1988458)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7879
diff
changeset
|
1084 |
attrs = {'tx_uuid': uuid, 'tx_user': ueid, 'tx_time': datetime.utcnow()} |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1085 |
self.doexec(cnx, self.sqlgen.insert('transactions', attrs), attrs) |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1086 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1087 |
def _save_attrs(self, cnx, entity, attrs): |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1088 |
"""return a pickleable dictionary containing current values for given |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1089 |
attributes of the entity |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1090 |
""" |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1091 |
restr = {'cw_eid': entity.eid} |
8900
010a59e12d89
use cw_etype instead of __regid__
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
8821
diff
changeset
|
1092 |
sql = self.sqlgen.select(SQL_PREFIX + entity.cw_etype, restr, attrs) |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1093 |
cu = self.doexec(cnx, sql, restr) |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1094 |
values = dict(zip(attrs, cu.fetchone())) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1095 |
# ensure backend specific binary are converted back to string |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1096 |
eschema = entity.e_schema |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1097 |
for column in attrs: |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1098 |
# [3:] remove 'cw_' prefix |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1099 |
attr = column[3:] |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1100 |
if not eschema.subjrels[attr].final: |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1101 |
continue |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1102 |
if eschema.destination(attr) in ('Password', 'Bytes'): |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1103 |
value = values[column] |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1104 |
if value is not None: |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1105 |
values[column] = self.binary_to_str(value) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1106 |
return values |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1107 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1108 |
def _record_tx_action(self, cnx, table, action, **kwargs): |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1109 |
"""record a transaction action in the given table (either |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1110 |
'tx_entity_actions' or 'tx_relation_action') |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1111 |
""" |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1112 |
kwargs['tx_uuid'] = cnx.transaction_uuid() |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1113 |
kwargs['txa_action'] = action |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1114 |
kwargs['txa_order'] = cnx.transaction_inc_action_counter() |
10351
91e63306e277
[connection] replace .running_dbapi_query with .hooks_in_progress
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10301
diff
changeset
|
1115 |
kwargs['txa_public'] = not cnx.hooks_in_progress |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1116 |
self.doexec(cnx, self.sqlgen.insert(table, kwargs), kwargs) |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1117 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1118 |
def _tx_info(self, cnx, txuuid): |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1119 |
"""return transaction's time and user of the transaction with the given uuid. |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1120 |
|
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1121 |
raise `NoSuchTransaction` if there is no such transaction of if the |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1122 |
connection's user isn't allowed to see it. |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1123 |
""" |
10365
21461f80f348
[connection] remove ensure_cnx_set context manager uses
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10364
diff
changeset
|
1124 |
restr = {'tx_uuid': txuuid} |
21461f80f348
[connection] remove ensure_cnx_set context manager uses
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10364
diff
changeset
|
1125 |
sql = self.sqlgen.select('transactions', restr, |
21461f80f348
[connection] remove ensure_cnx_set context manager uses
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10364
diff
changeset
|
1126 |
('tx_time', 'tx_user')) |
21461f80f348
[connection] remove ensure_cnx_set context manager uses
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10364
diff
changeset
|
1127 |
cu = self.doexec(cnx, sql, restr) |
21461f80f348
[connection] remove ensure_cnx_set context manager uses
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10364
diff
changeset
|
1128 |
try: |
21461f80f348
[connection] remove ensure_cnx_set context manager uses
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10364
diff
changeset
|
1129 |
time, ueid = cu.fetchone() |
21461f80f348
[connection] remove ensure_cnx_set context manager uses
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10364
diff
changeset
|
1130 |
except TypeError: |
21461f80f348
[connection] remove ensure_cnx_set context manager uses
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10364
diff
changeset
|
1131 |
raise tx.NoSuchTransaction(txuuid) |
21461f80f348
[connection] remove ensure_cnx_set context manager uses
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10364
diff
changeset
|
1132 |
if not (cnx.user.is_in_group('managers') |
21461f80f348
[connection] remove ensure_cnx_set context manager uses
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10364
diff
changeset
|
1133 |
or cnx.user.eid == ueid): |
21461f80f348
[connection] remove ensure_cnx_set context manager uses
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10364
diff
changeset
|
1134 |
raise tx.NoSuchTransaction(txuuid) |
21461f80f348
[connection] remove ensure_cnx_set context manager uses
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10364
diff
changeset
|
1135 |
return time, ueid |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1136 |
|
8265
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1137 |
def _reedit_entity(self, entity, changes, err): |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1138 |
cnx = entity._cw |
8265
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1139 |
eid = entity.eid |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1140 |
entity.cw_edited = edited = EditedEntity(entity) |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1141 |
# check for schema changes, entities linked through inlined relation |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1142 |
# still exists, rewrap binary values |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1143 |
eschema = entity.e_schema |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1144 |
getrschema = eschema.subjrels |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1145 |
for column, value in changes.items(): |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1146 |
rtype = column[len(SQL_PREFIX):] |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1147 |
if rtype == "eid": |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1148 |
continue # XXX should even `eid` be stored in action changes? |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1149 |
try: |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1150 |
rschema = getrschema[rtype] |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1151 |
except KeyError: |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1152 |
err(cnx._("can't restore relation %(rtype)s of entity %(eid)s, " |
8265
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1153 |
"this relation does not exist in the schema anymore.") |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1154 |
% {'rtype': rtype, 'eid': eid}) |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1155 |
if not rschema.final: |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1156 |
if not rschema.inlined: |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1157 |
assert value is None |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1158 |
# rschema is an inlined relation |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1159 |
elif value is not None: |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1160 |
# not a deletion: we must put something in edited |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1161 |
try: |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1162 |
entity._cw.entity_from_eid(value) # check target exists |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1163 |
edited[rtype] = value |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1164 |
except UnknownEid: |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1165 |
err(cnx._("can't restore entity %(eid)s of type %(eschema)s, " |
8265
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1166 |
"target of %(rtype)s (eid %(value)s) does not exist any longer") |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1167 |
% locals()) |
10206
0ecd61f4bd37
[undo] when we can't undo an inlined relation change, point at None, not a broken eid
Julien Cristau <julien.cristau@logilab.fr>
parents:
10203
diff
changeset
|
1168 |
changes[column] = None |
8265
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1169 |
elif eschema.destination(rtype) in ('Bytes', 'Password'): |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1170 |
changes[column] = self._binary(value) |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1171 |
edited[rtype] = Binary(value) |
10676
9d6b79081bad
[py3k] use text/binary types from six
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10662
diff
changeset
|
1172 |
elif PY2 and isinstance(value, str): |
9d6b79081bad
[py3k] use text/binary types from six
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10662
diff
changeset
|
1173 |
edited[rtype] = text_type(value, cnx.encoding, 'replace') |
8265
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1174 |
else: |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1175 |
edited[rtype] = value |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1176 |
# This must only be done after init_entitiy_caches : defered in calling functions |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1177 |
# edited.check() |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1178 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1179 |
def _undo_d(self, cnx, action): |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1180 |
"""undo an entity deletion""" |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1181 |
errors = [] |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1182 |
err = errors.append |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1183 |
eid = action.eid |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1184 |
etype = action.etype |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1185 |
_ = cnx._ |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1186 |
# get an entity instance |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1187 |
try: |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1188 |
entity = self.repo.vreg['etypes'].etype_class(etype)(cnx) |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1189 |
except Exception: |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1190 |
err("can't restore entity %s of type %s, type no more supported" |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1191 |
% (eid, etype)) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1192 |
return errors |
8265
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1193 |
self._reedit_entity(entity, action.changes, err) |
5557
1a534c596bff
[entity] continue cleanup of Entity/AnyEntity namespace
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5556
diff
changeset
|
1194 |
entity.eid = eid |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1195 |
cnx.repo.init_entity_caches(cnx, entity, self) |
8265
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1196 |
entity.cw_edited.check() |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1197 |
self.repo.hm.call_hooks('before_add_entity', cnx, entity=entity) |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1198 |
# restore the entity |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1199 |
action.changes['cw_eid'] = eid |
10203
1443fe643a38
[server] change order of entities table modification vs entity creation/deletion
Julien Cristau <julien.cristau@logilab.fr>
parents:
10200
diff
changeset
|
1200 |
# restore record in entities (will update fti if needed) |
1443fe643a38
[server] change order of entities table modification vs entity creation/deletion
Julien Cristau <julien.cristau@logilab.fr>
parents:
10200
diff
changeset
|
1201 |
self.add_info(cnx, entity, self, None) |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1202 |
sql = self.sqlgen.insert(SQL_PREFIX + etype, action.changes) |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1203 |
self.doexec(cnx, sql, action.changes) |
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1204 |
self.repo.hm.call_hooks('after_add_entity', cnx, entity=entity) |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1205 |
return errors |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1206 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1207 |
def _undo_r(self, cnx, action): |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1208 |
"""undo a relation removal""" |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1209 |
errors = [] |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1210 |
subj, rtype, obj = action.eid_from, action.rtype, action.eid_to |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1211 |
try: |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1212 |
sentity, oentity, rdef = _undo_rel_info(cnx, subj, rtype, obj) |
8695
358d8bed9626
[toward-py3k] rewrite to "except AnException as exc:" (part of #2711624)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
8694
diff
changeset
|
1213 |
except _UndoException as ex: |
10676
9d6b79081bad
[py3k] use text/binary types from six
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10662
diff
changeset
|
1214 |
errors.append(text_type(ex)) |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1215 |
else: |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1216 |
for role, entity in (('subject', sentity), |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1217 |
('object', oentity)): |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1218 |
try: |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1219 |
_undo_check_relation_target(entity, rdef, role) |
8695
358d8bed9626
[toward-py3k] rewrite to "except AnException as exc:" (part of #2711624)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
8694
diff
changeset
|
1220 |
except _UndoException as ex: |
10676
9d6b79081bad
[py3k] use text/binary types from six
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10662
diff
changeset
|
1221 |
errors.append(text_type(ex)) |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1222 |
continue |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1223 |
if not errors: |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1224 |
self.repo.hm.call_hooks('before_add_relation', cnx, |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1225 |
eidfrom=subj, rtype=rtype, eidto=obj) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1226 |
# add relation in the database |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1227 |
self._add_relations(cnx, rtype, [(subj, obj)], rdef.rtype.inlined) |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1228 |
# set related cache |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1229 |
cnx.update_rel_cache_add(subj, rtype, obj, rdef.rtype.symmetric) |
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1230 |
self.repo.hm.call_hooks('after_add_relation', cnx, |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1231 |
eidfrom=subj, rtype=rtype, eidto=obj) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1232 |
return errors |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1233 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1234 |
def _undo_c(self, cnx, action): |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1235 |
"""undo an entity creation""" |
5076
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1236 |
eid = action.eid |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1237 |
# XXX done to avoid fetching all remaining relation for the entity |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1238 |
# we should find an efficient way to do this (keeping current veolidf |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1239 |
# massive deletion performance) |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1240 |
if _undo_has_later_transaction(cnx, eid): |
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1241 |
msg = cnx._('some later transaction(s) touch entity, undo them ' |
5076
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1242 |
'first') |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1243 |
raise ValidationError(eid, {None: msg}) |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1244 |
etype = action.etype |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1245 |
# get an entity instance |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1246 |
try: |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1247 |
entity = self.repo.vreg['etypes'].etype_class(etype)(cnx) |
5076
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1248 |
except Exception: |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1249 |
return [cnx._( |
5098
32b1adfb6b92
[i18n] use named substitution to avoid gettext warnings
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5076
diff
changeset
|
1250 |
"Can't undo creation of entity %(eid)s of type %(etype)s, type " |
32b1adfb6b92
[i18n] use named substitution to avoid gettext warnings
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5076
diff
changeset
|
1251 |
"no more supported" % {'eid': eid, 'etype': etype})] |
5557
1a534c596bff
[entity] continue cleanup of Entity/AnyEntity namespace
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5556
diff
changeset
|
1252 |
entity.eid = eid |
5076
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1253 |
# for proper eid/type cache update |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1254 |
CleanupDeletedEidsCacheOp.get_instance(cnx).add_data(eid) |
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1255 |
self.repo.hm.call_hooks('before_delete_entity', cnx, entity=entity) |
5076
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1256 |
# remove is / is_instance_of which are added using sql by hooks, hence |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1257 |
# unvisible as transaction action |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1258 |
self.doexec(cnx, 'DELETE FROM is_relation WHERE eid_from=%s' % eid) |
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1259 |
self.doexec(cnx, 'DELETE FROM is_instance_of_relation WHERE eid_from=%s' % eid) |
10183
ff93dad2ae3a
[undo] fix deletion of cw_source relation
Julien Cristau <julien.cristau@logilab.fr>
parents:
10178
diff
changeset
|
1260 |
self.doexec(cnx, 'DELETE FROM cw_source_relation WHERE eid_from=%s' % eid) |
5076
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1261 |
# XXX check removal of inlined relation? |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1262 |
# delete the entity |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1263 |
attrs = {'cw_eid': eid} |
8900
010a59e12d89
use cw_etype instead of __regid__
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
8821
diff
changeset
|
1264 |
sql = self.sqlgen.delete(SQL_PREFIX + entity.cw_etype, attrs) |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1265 |
self.doexec(cnx, sql, attrs) |
5076
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1266 |
# remove record from entities (will update fti if needed) |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1267 |
self.delete_info_multi(cnx, [entity]) |
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1268 |
self.repo.hm.call_hooks('after_delete_entity', cnx, entity=entity) |
5076
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1269 |
return () |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1270 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1271 |
def _undo_u(self, cnx, action): |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1272 |
"""undo an entity update""" |
8265
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1273 |
errors = [] |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1274 |
err = errors.append |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1275 |
try: |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1276 |
entity = cnx.entity_from_eid(action.eid) |
8265
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1277 |
except UnknownEid: |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1278 |
err(cnx._("can't restore state of entity %s, it has been " |
8265
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1279 |
"deleted inbetween") % action.eid) |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1280 |
return errors |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1281 |
self._reedit_entity(entity, action.changes, err) |
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1282 |
entity.cw_edited.check() |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1283 |
self.repo.hm.call_hooks('before_update_entity', cnx, entity=entity) |
8900
010a59e12d89
use cw_etype instead of __regid__
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
8821
diff
changeset
|
1284 |
sql = self.sqlgen.update(SQL_PREFIX + entity.cw_etype, action.changes, |
8265
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1285 |
['cw_eid']) |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1286 |
self.doexec(cnx, sql, action.changes) |
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1287 |
self.repo.hm.call_hooks('after_update_entity', cnx, entity=entity) |
8265
9747ab9230ad
[repo, undo] Finish repository-side implementation of the undo feature (closes #893940)
Anthony Truchet <anthony.truchet@logilab.fr>
parents:
8235
diff
changeset
|
1288 |
return errors |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1289 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1290 |
def _undo_a(self, cnx, action): |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1291 |
"""undo a relation addition""" |
5076
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1292 |
errors = [] |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1293 |
subj, rtype, obj = action.eid_from, action.rtype, action.eid_to |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1294 |
try: |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1295 |
sentity, oentity, rdef = _undo_rel_info(cnx, subj, rtype, obj) |
8695
358d8bed9626
[toward-py3k] rewrite to "except AnException as exc:" (part of #2711624)
Nicolas Chauvat <nicolas.chauvat@logilab.fr>
parents:
8694
diff
changeset
|
1296 |
except _UndoException as ex: |
10676
9d6b79081bad
[py3k] use text/binary types from six
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10662
diff
changeset
|
1297 |
errors.append(text_type(ex)) |
5076
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1298 |
else: |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1299 |
rschema = rdef.rtype |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1300 |
if rschema.inlined: |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1301 |
sql = 'SELECT 1 FROM cw_%s WHERE cw_eid=%s and cw_%s=%s'\ |
8900
010a59e12d89
use cw_etype instead of __regid__
Pierre-Yves David <pierre-yves.david@logilab.fr>
parents:
8821
diff
changeset
|
1302 |
% (sentity.cw_etype, subj, rtype, obj) |
5076
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1303 |
else: |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1304 |
sql = 'SELECT 1 FROM %s_relation WHERE eid_from=%s and eid_to=%s'\ |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1305 |
% (rtype, subj, obj) |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1306 |
cu = self.doexec(cnx, sql) |
5076
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1307 |
if cu.fetchone() is None: |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1308 |
errors.append(cnx._( |
5098
32b1adfb6b92
[i18n] use named substitution to avoid gettext warnings
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5076
diff
changeset
|
1309 |
"Can't undo addition of relation %(rtype)s from %(subj)s to" |
32b1adfb6b92
[i18n] use named substitution to avoid gettext warnings
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5076
diff
changeset
|
1310 |
" %(obj)s, doesn't exist anymore" % locals())) |
5076
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1311 |
if not errors: |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1312 |
self.repo.hm.call_hooks('before_delete_relation', cnx, |
5076
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1313 |
eidfrom=subj, rtype=rtype, eidto=obj) |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1314 |
# delete relation from the database |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1315 |
self._delete_relation(cnx, subj, rtype, obj, rschema.inlined) |
5076
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1316 |
# set related cache |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1317 |
cnx.update_rel_cache_del(subj, rtype, obj, rschema.symmetric) |
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1318 |
self.repo.hm.call_hooks('after_delete_relation', cnx, |
5076
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1319 |
eidfrom=subj, rtype=rtype, eidto=obj) |
b0e6134b4324
[undo] basic support for undoing of entity creation / relation addition
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5075
diff
changeset
|
1320 |
return errors |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1321 |
|
4806
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
1322 |
# full text index handling ################################################# |
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
1323 |
|
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
1324 |
@cached |
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
1325 |
def need_fti_indexation(self, etype): |
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
1326 |
eschema = self.schema.eschema(etype) |
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
1327 |
if any(eschema.indexable_attributes()): |
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
1328 |
return True |
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
1329 |
if any(eschema.fulltext_containers()): |
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
1330 |
return True |
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
1331 |
return False |
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
1332 |
|
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1333 |
def index_entity(self, cnx, entity): |
4807
5642bfa43236
[cleanup] add index_entity to abstract source, add docstring
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4806
diff
changeset
|
1334 |
"""create an operation to [re]index textual content of the given entity |
5642bfa43236
[cleanup] add index_entity to abstract source, add docstring
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4806
diff
changeset
|
1335 |
on commit |
5642bfa43236
[cleanup] add index_entity to abstract source, add docstring
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4806
diff
changeset
|
1336 |
""" |
10631
1ab79c435fae
consider .do_fti flag in index_entity method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10329
diff
changeset
|
1337 |
if self.do_fti: |
1ab79c435fae
consider .do_fti flag in index_entity method
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
10329
diff
changeset
|
1338 |
FTIndexEntityOp.get_instance(cnx).add_data(entity.eid) |
4806
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
1339 |
|
9574
2d4c4842bd04
[server] some s/session/cnx/
Julien Cristau <julien.cristau@logilab.fr>
parents:
9546
diff
changeset
|
1340 |
def fti_unindex_entities(self, cnx, entities): |
6889
37668bf302f5
improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6850
diff
changeset
|
1341 |
"""remove text content for entities from the full text index |
4806
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
1342 |
""" |
9574
2d4c4842bd04
[server] some s/session/cnx/
Julien Cristau <julien.cristau@logilab.fr>
parents:
9546
diff
changeset
|
1343 |
cursor = cnx.cnxset.cu |
6889
37668bf302f5
improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6850
diff
changeset
|
1344 |
cursor_unindex_object = self.dbhelper.cursor_unindex_object |
4806
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
1345 |
try: |
6889
37668bf302f5
improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6850
diff
changeset
|
1346 |
for entity in entities: |
37668bf302f5
improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6850
diff
changeset
|
1347 |
cursor_unindex_object(entity.eid, cursor) |
4806
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
1348 |
except Exception: # let KeyboardInterrupt / SystemExit propagate |
6889
37668bf302f5
improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6850
diff
changeset
|
1349 |
self.exception('error while unindexing %s', entity) |
37668bf302f5
improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6850
diff
changeset
|
1350 |
|
4806
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
1351 |
|
9574
2d4c4842bd04
[server] some s/session/cnx/
Julien Cristau <julien.cristau@logilab.fr>
parents:
9546
diff
changeset
|
1352 |
def fti_index_entities(self, cnx, entities): |
6889
37668bf302f5
improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6850
diff
changeset
|
1353 |
"""add text content of created/modified entities to the full text index |
4806
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
1354 |
""" |
6889
37668bf302f5
improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6850
diff
changeset
|
1355 |
cursor_index_object = self.dbhelper.cursor_index_object |
9574
2d4c4842bd04
[server] some s/session/cnx/
Julien Cristau <julien.cristau@logilab.fr>
parents:
9546
diff
changeset
|
1356 |
cursor = cnx.cnxset.cu |
4806
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
1357 |
try: |
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
1358 |
# use cursor_index_object, not cursor_reindex_object since |
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
1359 |
# unindexing done in the FTIndexEntityOp |
6889
37668bf302f5
improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6850
diff
changeset
|
1360 |
for entity in entities: |
37668bf302f5
improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6850
diff
changeset
|
1361 |
cursor_index_object(entity.eid, |
37668bf302f5
improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6850
diff
changeset
|
1362 |
entity.cw_adapt_to('IFTIndexable'), |
37668bf302f5
improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6850
diff
changeset
|
1363 |
cursor) |
4806
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
1364 |
except Exception: # let KeyboardInterrupt / SystemExit propagate |
6889
37668bf302f5
improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6850
diff
changeset
|
1365 |
self.exception('error while indexing %s', entity) |
4806
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
1366 |
|
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
1367 |
|
6426
541659c39f6a
[hook/operation] nicer api to achieve same result as set_operation, as described in #1253630
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6366
diff
changeset
|
1368 |
class FTIndexEntityOp(hook.DataOperationMixIn, hook.LateOperation): |
4806
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
1369 |
"""operation to delay entity full text indexation to commit |
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
1370 |
|
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
1371 |
since fti indexing may trigger discovery of other entities, it should be |
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
1372 |
triggered on precommit, not commit, and this should be done after other |
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
1373 |
precommit operation which may add relations to the entity |
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
1374 |
""" |
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
1375 |
|
4f12f59b1a13
[fti] refactor and fix full text indexation handling
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4692
diff
changeset
|
1376 |
def precommit_event(self): |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1377 |
cnx = self.cnx |
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1378 |
source = cnx.repo.system_source |
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1379 |
pendingeids = cnx.transaction_data.get('pendingeids', ()) |
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1380 |
done = cnx.transaction_data.setdefault('indexedeids', set()) |
6889
37668bf302f5
improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6850
diff
changeset
|
1381 |
to_reindex = set() |
6426
541659c39f6a
[hook/operation] nicer api to achieve same result as set_operation, as described in #1253630
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6366
diff
changeset
|
1382 |
for eid in self.get_data(): |
5070
b1f80ccadda3
[repo] refactor fti operation to use set_operation and a single operation whatever the number of entities to ft index
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5067
diff
changeset
|
1383 |
if eid in pendingeids or eid in done: |
b1f80ccadda3
[repo] refactor fti operation to use set_operation and a single operation whatever the number of entities to ft index
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5067
diff
changeset
|
1384 |
# entity added and deleted in the same transaction or already |
b1f80ccadda3
[repo] refactor fti operation to use set_operation and a single operation whatever the number of entities to ft index
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5067
diff
changeset
|
1385 |
# processed |
6889
37668bf302f5
improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6850
diff
changeset
|
1386 |
continue |
5070
b1f80ccadda3
[repo] refactor fti operation to use set_operation and a single operation whatever the number of entities to ft index
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5067
diff
changeset
|
1387 |
done.add(eid) |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1388 |
iftindexable = cnx.entity_from_eid(eid).cw_adapt_to('IFTIndexable') |
6889
37668bf302f5
improve massive deletion performance
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
6850
diff
changeset
|
1389 |
to_reindex |= set(iftindexable.fti_containers()) |
9611
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1390 |
source.fti_unindex_entities(cnx, to_reindex) |
ca853478aaa8
[server/sources/native] deal with connections, not sessions
Julien Cristau <julien.cristau@logilab.fr>
parents:
9605
diff
changeset
|
1391 |
source.fti_index_entities(cnx, to_reindex) |
0 | 1392 |
|
1393 |
def sql_schema(driver): |
|
4831
c5aec27c1bf7
[repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4829
diff
changeset
|
1394 |
helper = get_db_helper(driver) |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1395 |
typemap = helper.TYPE_MAPPING |
0 | 1396 |
schema = """ |
1397 |
/* Create the repository's system database */ |
|
1398 |
||
1399 |
%s |
|
1400 |
||
1401 |
CREATE TABLE entities ( |
|
1402 |
eid INTEGER PRIMARY KEY NOT NULL, |
|
1403 |
type VARCHAR(64) NOT NULL, |
|
8205
14c52c5ee706
[entities system table] change asource and source column size (closes #1951819)
Alexandre Richardson <alexandre.richardson@logilab.fr>
parents:
7967
diff
changeset
|
1404 |
asource VARCHAR(128) NOT NULL, |
0 | 1405 |
extid VARCHAR(256) |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1406 |
);; |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1407 |
CREATE INDEX entities_type_idx ON entities(type);; |
10209
4c64a41c0a1d
Use a moved_entities table to record external entities moved to the system source
Julien Cristau <julien.cristau@logilab.fr>
parents:
10208
diff
changeset
|
1408 |
CREATE TABLE moved_entities ( |
4c64a41c0a1d
Use a moved_entities table to record external entities moved to the system source
Julien Cristau <julien.cristau@logilab.fr>
parents:
10208
diff
changeset
|
1409 |
eid INTEGER PRIMARY KEY NOT NULL, |
10627
ffef8ac383f4
moved_entities.extid may not be NULL
Julien Cristau <julien.cristau@logilab.fr>
parents:
10626
diff
changeset
|
1410 |
extid VARCHAR(256) UNIQUE NOT NULL |
10209
4c64a41c0a1d
Use a moved_entities table to record external entities moved to the system source
Julien Cristau <julien.cristau@logilab.fr>
parents:
10208
diff
changeset
|
1411 |
);; |
0 | 1412 |
|
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1413 |
CREATE TABLE transactions ( |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1414 |
tx_uuid CHAR(32) PRIMARY KEY NOT NULL, |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1415 |
tx_user INTEGER NOT NULL, |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1416 |
tx_time %s NOT NULL |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1417 |
);; |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1418 |
CREATE INDEX transactions_tx_user_idx ON transactions(tx_user);; |
9336
722635e530a0
[server] Add missing indices for undo support (closes #3259691)
Julien Cristau <julien.cristau@logilab.fr>
parents:
9298
diff
changeset
|
1419 |
CREATE INDEX transactions_tx_time_idx ON transactions(tx_time);; |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1420 |
|
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1421 |
CREATE TABLE tx_entity_actions ( |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1422 |
tx_uuid CHAR(32) REFERENCES transactions(tx_uuid) ON DELETE CASCADE, |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1423 |
txa_action CHAR(1) NOT NULL, |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1424 |
txa_public %s NOT NULL, |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1425 |
txa_order INTEGER, |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1426 |
eid INTEGER NOT NULL, |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1427 |
etype VARCHAR(64) NOT NULL, |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1428 |
changes %s |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1429 |
);; |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1430 |
CREATE INDEX tx_entity_actions_txa_action_idx ON tx_entity_actions(txa_action);; |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1431 |
CREATE INDEX tx_entity_actions_txa_public_idx ON tx_entity_actions(txa_public);; |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1432 |
CREATE INDEX tx_entity_actions_eid_idx ON tx_entity_actions(eid);; |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1433 |
CREATE INDEX tx_entity_actions_etype_idx ON tx_entity_actions(etype);; |
9336
722635e530a0
[server] Add missing indices for undo support (closes #3259691)
Julien Cristau <julien.cristau@logilab.fr>
parents:
9298
diff
changeset
|
1434 |
CREATE INDEX tx_entity_actions_tx_uuid_idx ON tx_entity_actions(tx_uuid);; |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1435 |
|
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1436 |
CREATE TABLE tx_relation_actions ( |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1437 |
tx_uuid CHAR(32) REFERENCES transactions(tx_uuid) ON DELETE CASCADE, |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1438 |
txa_action CHAR(1) NOT NULL, |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1439 |
txa_public %s NOT NULL, |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1440 |
txa_order INTEGER, |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1441 |
eid_from INTEGER NOT NULL, |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1442 |
eid_to INTEGER NOT NULL, |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1443 |
rtype VARCHAR(256) NOT NULL |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1444 |
);; |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1445 |
CREATE INDEX tx_relation_actions_txa_action_idx ON tx_relation_actions(txa_action);; |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1446 |
CREATE INDEX tx_relation_actions_txa_public_idx ON tx_relation_actions(txa_public);; |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1447 |
CREATE INDEX tx_relation_actions_eid_from_idx ON tx_relation_actions(eid_from);; |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1448 |
CREATE INDEX tx_relation_actions_eid_to_idx ON tx_relation_actions(eid_to);; |
9336
722635e530a0
[server] Add missing indices for undo support (closes #3259691)
Julien Cristau <julien.cristau@logilab.fr>
parents:
9298
diff
changeset
|
1449 |
CREATE INDEX tx_relation_actions_tx_uuid_idx ON tx_relation_actions(tx_uuid);; |
9585
3f5b59527d31
[source/native] allow many eid creation per .create_eid call (closes #3526594)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9583
diff
changeset
|
1450 |
""" % (helper.sql_create_numrange('entities_id_seq').replace(';', ';;'), |
9450
af4b93bc38a5
[multi-sources-removal] Drop deleted_entities system table and entities.mtime column
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9448
diff
changeset
|
1451 |
typemap['Datetime'], |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1452 |
typemap['Boolean'], typemap['Bytes'], typemap['Boolean']) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1453 |
if helper.backend_name == 'sqlite': |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1454 |
# sqlite support the ON DELETE CASCADE syntax but do nothing |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1455 |
schema += ''' |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1456 |
CREATE TRIGGER fkd_transactions |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1457 |
BEFORE DELETE ON transactions |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1458 |
FOR EACH ROW BEGIN |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1459 |
DELETE FROM tx_entity_actions WHERE tx_uuid=OLD.tx_uuid; |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1460 |
DELETE FROM tx_relation_actions WHERE tx_uuid=OLD.tx_uuid; |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1461 |
END;; |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1462 |
''' |
10626
93f8d239c708
[server] use sqls_create_multicol_unique_index to index entities.extid
Julien Cristau <julien.cristau@logilab.fr>
parents:
10575
diff
changeset
|
1463 |
schema += ';;'.join(helper.sqls_create_multicol_unique_index('entities', ['extid'])) |
93f8d239c708
[server] use sqls_create_multicol_unique_index to index entities.extid
Julien Cristau <julien.cristau@logilab.fr>
parents:
10575
diff
changeset
|
1464 |
schema += ';;\n' |
0 | 1465 |
return schema |
1466 |
||
1467 |
||
1468 |
def sql_drop_schema(driver): |
|
4831
c5aec27c1bf7
[repo] use logilab.db instead of lgc.adbh/lgc.db/lgc.sqlgen/indexer, test new date extranction functions
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
4829
diff
changeset
|
1469 |
helper = get_db_helper(driver) |
0 | 1470 |
return """ |
10626
93f8d239c708
[server] use sqls_create_multicol_unique_index to index entities.extid
Julien Cristau <julien.cristau@logilab.fr>
parents:
10575
diff
changeset
|
1471 |
%s; |
0 | 1472 |
%s |
1473 |
DROP TABLE entities; |
|
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1474 |
DROP TABLE tx_entity_actions; |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1475 |
DROP TABLE tx_relation_actions; |
5214
3285b6e3b930
fix cwctl db-init -d on SQL Server
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
5168
diff
changeset
|
1476 |
DROP TABLE transactions; |
10626
93f8d239c708
[server] use sqls_create_multicol_unique_index to index entities.extid
Julien Cristau <julien.cristau@logilab.fr>
parents:
10575
diff
changeset
|
1477 |
""" % (';'.join(helper.sqls_drop_multicol_unique_index('entities', ['extid'])), |
93f8d239c708
[server] use sqls_create_multicol_unique_index to index entities.extid
Julien Cristau <julien.cristau@logilab.fr>
parents:
10575
diff
changeset
|
1478 |
helper.sql_drop_numrange('entities_id_seq')) |
0 | 1479 |
|
1480 |
||
1481 |
def grant_schema(user, set_owner=True): |
|
1482 |
result = '' |
|
9450
af4b93bc38a5
[multi-sources-removal] Drop deleted_entities system table and entities.mtime column
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
9448
diff
changeset
|
1483 |
for table in ('entities', 'entities_id_seq', |
4913
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1484 |
'transactions', 'tx_entity_actions', 'tx_relation_actions'): |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1485 |
if set_owner: |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1486 |
result = 'ALTER TABLE %s OWNER TO %s;\n' % (table, user) |
083b4d454192
server/web api for accessing to deleted_entites
Katia Saurfelt <katia.saurfelt@logilab.fr>
parents:
4902
diff
changeset
|
1487 |
result += 'GRANT ALL ON %s TO %s;\n' % (table, user) |
0 | 1488 |
return result |
3647
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1489 |
|
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1490 |
|
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1491 |
class BaseAuthentifier(object): |
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1492 |
|
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1493 |
def __init__(self, source=None): |
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1494 |
self.source = source |
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1495 |
|
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1496 |
def set_schema(self, schema): |
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1497 |
"""set the instance'schema""" |
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1498 |
pass |
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1499 |
|
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1500 |
class LoginPasswordAuthentifier(BaseAuthentifier): |
8520
fcd048fa6e6d
[source/native] only system source users should be authenticated by the native source (closes #2465883)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8508
diff
changeset
|
1501 |
passwd_rql = 'Any P WHERE X is CWUser, X login %(login)s, X upassword P' |
10095
200bd6a601dc
[warnings] put an end to warnings in the sqlite driver over `str` being sent instead of unicode strings
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10086
diff
changeset
|
1502 |
auth_rql = (u'Any X WHERE X is CWUser, X login %(login)s, X upassword %(pwd)s, ' |
8520
fcd048fa6e6d
[source/native] only system source users should be authenticated by the native source (closes #2465883)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8508
diff
changeset
|
1503 |
'X cw_source S, S name "system"') |
fcd048fa6e6d
[source/native] only system source users should be authenticated by the native source (closes #2465883)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8508
diff
changeset
|
1504 |
_sols = ({'X': 'CWUser', 'P': 'Password', 'S': 'CWSource'},) |
3647
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1505 |
|
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1506 |
def set_schema(self, schema): |
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1507 |
"""set the instance'schema""" |
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1508 |
if 'CWUser' in schema: # probably an empty schema if not true... |
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1509 |
# rql syntax trees used to authenticate users |
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1510 |
self._passwd_rqlst = self.source.compile_rql(self.passwd_rql, self._sols) |
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1511 |
self._auth_rqlst = self.source.compile_rql(self.auth_rql, self._sols) |
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1512 |
|
9512
88dc96fc9fc1
[server] use a connection instead of a session for user authentication
Julien Cristau <julien.cristau@logilab.fr>
parents:
9487
diff
changeset
|
1513 |
def authenticate(self, cnx, login, password=None, **kwargs): |
3647
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1514 |
"""return CWUser eid for the given login/password if this account is |
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1515 |
defined in this source, else raise `AuthenticationError` |
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1516 |
|
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1517 |
two queries are needed since passwords are stored crypted, so we have |
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1518 |
to fetch the salt first |
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1519 |
""" |
6018
f4d1d5d9ccbb
[security] don't put uncrypted password in query parameters, else it may be logged on error
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
5975
diff
changeset
|
1520 |
args = {'login': login, 'pwd' : None} |
3647
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1521 |
if password is not None: |
9512
88dc96fc9fc1
[server] use a connection instead of a session for user authentication
Julien Cristau <julien.cristau@logilab.fr>
parents:
9487
diff
changeset
|
1522 |
rset = self.source.syntax_tree_search(cnx, self._passwd_rqlst, args) |
3647
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1523 |
try: |
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1524 |
pwd = rset[0][0] |
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1525 |
except IndexError: |
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1526 |
raise AuthenticationError('bad login') |
8235
c2a91d6639d8
[system source] fix authentication: don't allow login whatever the given password if user password is NULL in database. Closes #2186099
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8205
diff
changeset
|
1527 |
if pwd is None: |
c2a91d6639d8
[system source] fix authentication: don't allow login whatever the given password if user password is NULL in database. Closes #2186099
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8205
diff
changeset
|
1528 |
# if pwd is None but a password is provided, something is wrong |
c2a91d6639d8
[system source] fix authentication: don't allow login whatever the given password if user password is NULL in database. Closes #2186099
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8205
diff
changeset
|
1529 |
raise AuthenticationError('bad password') |
3647
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1530 |
# passwords are stored using the Bytes type, so we get a StringIO |
8317
9c59258e7798
[security] use a stronger encryption algorythm for password, keeping bw compat
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8235
diff
changeset
|
1531 |
args['pwd'] = Binary(crypt_password(password, pwd.getvalue())) |
3647
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1532 |
# get eid from login and (crypted) password |
9512
88dc96fc9fc1
[server] use a connection instead of a session for user authentication
Julien Cristau <julien.cristau@logilab.fr>
parents:
9487
diff
changeset
|
1533 |
rset = self.source.syntax_tree_search(cnx, self._auth_rqlst, args) |
10037
6b83ceda8323
[server] fix LoginPasswordAuthentifier.authenticate
David Douard <david.douard@logilab.fr>
parents:
9996
diff
changeset
|
1534 |
pwd = args['pwd'] |
3647
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1535 |
try: |
8546
3d2038d6f20d
[sources/native] automatically update passwords using deprecated hashes on login
Julien Cristau <julien.cristau@logilab.fr>
parents:
8520
diff
changeset
|
1536 |
user = rset[0][0] |
3d2038d6f20d
[sources/native] automatically update passwords using deprecated hashes on login
Julien Cristau <julien.cristau@logilab.fr>
parents:
8520
diff
changeset
|
1537 |
# If the stored hash uses a deprecated scheme (e.g. DES or MD5 used |
3d2038d6f20d
[sources/native] automatically update passwords using deprecated hashes on login
Julien Cristau <julien.cristau@logilab.fr>
parents:
8520
diff
changeset
|
1538 |
# before 3.14.7), update with a fresh one |
10037
6b83ceda8323
[server] fix LoginPasswordAuthentifier.authenticate
David Douard <david.douard@logilab.fr>
parents:
9996
diff
changeset
|
1539 |
if pwd is not None and pwd.getvalue(): |
8546
3d2038d6f20d
[sources/native] automatically update passwords using deprecated hashes on login
Julien Cristau <julien.cristau@logilab.fr>
parents:
8520
diff
changeset
|
1540 |
verify, newhash = verify_and_update(password, pwd.getvalue()) |
3d2038d6f20d
[sources/native] automatically update passwords using deprecated hashes on login
Julien Cristau <julien.cristau@logilab.fr>
parents:
8520
diff
changeset
|
1541 |
if not verify: # should not happen, but... |
3d2038d6f20d
[sources/native] automatically update passwords using deprecated hashes on login
Julien Cristau <julien.cristau@logilab.fr>
parents:
8520
diff
changeset
|
1542 |
raise AuthenticationError('bad password') |
3d2038d6f20d
[sources/native] automatically update passwords using deprecated hashes on login
Julien Cristau <julien.cristau@logilab.fr>
parents:
8520
diff
changeset
|
1543 |
if newhash: |
9512
88dc96fc9fc1
[server] use a connection instead of a session for user authentication
Julien Cristau <julien.cristau@logilab.fr>
parents:
9487
diff
changeset
|
1544 |
cnx.system_sql("UPDATE %s SET %s=%%(newhash)s WHERE %s=%%(login)s" % ( |
8546
3d2038d6f20d
[sources/native] automatically update passwords using deprecated hashes on login
Julien Cristau <julien.cristau@logilab.fr>
parents:
8520
diff
changeset
|
1545 |
SQL_PREFIX + 'CWUser', |
3d2038d6f20d
[sources/native] automatically update passwords using deprecated hashes on login
Julien Cristau <julien.cristau@logilab.fr>
parents:
8520
diff
changeset
|
1546 |
SQL_PREFIX + 'upassword', |
3d2038d6f20d
[sources/native] automatically update passwords using deprecated hashes on login
Julien Cristau <julien.cristau@logilab.fr>
parents:
8520
diff
changeset
|
1547 |
SQL_PREFIX + 'login'), |
3d2038d6f20d
[sources/native] automatically update passwords using deprecated hashes on login
Julien Cristau <julien.cristau@logilab.fr>
parents:
8520
diff
changeset
|
1548 |
{'newhash': self.source._binary(newhash), |
3d2038d6f20d
[sources/native] automatically update passwords using deprecated hashes on login
Julien Cristau <julien.cristau@logilab.fr>
parents:
8520
diff
changeset
|
1549 |
'login': login}) |
10364
8b35a898b334
[server] remove cnxset tracking, it is now unneeded
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10360
diff
changeset
|
1550 |
cnx.commit() |
8546
3d2038d6f20d
[sources/native] automatically update passwords using deprecated hashes on login
Julien Cristau <julien.cristau@logilab.fr>
parents:
8520
diff
changeset
|
1551 |
return user |
3647
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1552 |
except IndexError: |
2941f4a0aab9
refactor repo authentication to allow pluggable authentifier to login with something else than a password
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
3503
diff
changeset
|
1553 |
raise AuthenticationError('bad password') |
6849
5a0c2cfc19bf
[repository auth] cleanup email login by turning it into a proper repo-side authentication plugin
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6724
diff
changeset
|
1554 |
|
5a0c2cfc19bf
[repository auth] cleanup email login by turning it into a proper repo-side authentication plugin
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6724
diff
changeset
|
1555 |
|
5a0c2cfc19bf
[repository auth] cleanup email login by turning it into a proper repo-side authentication plugin
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6724
diff
changeset
|
1556 |
class EmailPasswordAuthentifier(BaseAuthentifier): |
9512
88dc96fc9fc1
[server] use a connection instead of a session for user authentication
Julien Cristau <julien.cristau@logilab.fr>
parents:
9487
diff
changeset
|
1557 |
def authenticate(self, cnx, login, **authinfo): |
6849
5a0c2cfc19bf
[repository auth] cleanup email login by turning it into a proper repo-side authentication plugin
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6724
diff
changeset
|
1558 |
# email_auth flag prevent from infinite recursion (call to |
5a0c2cfc19bf
[repository auth] cleanup email login by turning it into a proper repo-side authentication plugin
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6724
diff
changeset
|
1559 |
# repo.check_auth_info at the end of this method may lead us here again) |
5a0c2cfc19bf
[repository auth] cleanup email login by turning it into a proper repo-side authentication plugin
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6724
diff
changeset
|
1560 |
if not '@' in login or authinfo.pop('email_auth', None): |
5a0c2cfc19bf
[repository auth] cleanup email login by turning it into a proper repo-side authentication plugin
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6724
diff
changeset
|
1561 |
raise AuthenticationError('not an email') |
9512
88dc96fc9fc1
[server] use a connection instead of a session for user authentication
Julien Cristau <julien.cristau@logilab.fr>
parents:
9487
diff
changeset
|
1562 |
rset = cnx.execute('Any L WHERE U login L, U primary_email M, ' |
6849
5a0c2cfc19bf
[repository auth] cleanup email login by turning it into a proper repo-side authentication plugin
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6724
diff
changeset
|
1563 |
'M address %(login)s', {'login': login}, |
5a0c2cfc19bf
[repository auth] cleanup email login by turning it into a proper repo-side authentication plugin
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6724
diff
changeset
|
1564 |
build_descr=False) |
5a0c2cfc19bf
[repository auth] cleanup email login by turning it into a proper repo-side authentication plugin
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6724
diff
changeset
|
1565 |
if rset.rowcount != 1: |
5a0c2cfc19bf
[repository auth] cleanup email login by turning it into a proper repo-side authentication plugin
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6724
diff
changeset
|
1566 |
raise AuthenticationError('unexisting email') |
5a0c2cfc19bf
[repository auth] cleanup email login by turning it into a proper repo-side authentication plugin
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6724
diff
changeset
|
1567 |
login = rset.rows[0][0] |
5a0c2cfc19bf
[repository auth] cleanup email login by turning it into a proper repo-side authentication plugin
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
6724
diff
changeset
|
1568 |
authinfo['email_auth'] = True |
9512
88dc96fc9fc1
[server] use a connection instead of a session for user authentication
Julien Cristau <julien.cristau@logilab.fr>
parents:
9487
diff
changeset
|
1569 |
return self.source.repo.check_auth_info(cnx, login, authinfo) |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1570 |
|
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1571 |
|
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1572 |
class DatabaseIndependentBackupRestore(object): |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1573 |
"""Helper class to perform db backend agnostic backup and restore |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1574 |
|
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1575 |
The backup and restore methods are used to dump / restore the |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1576 |
system database in a database independent format. The file is a |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1577 |
Zip archive containing the following files: |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1578 |
|
10197
7c732d2abbc1
[restore/portable] properly restore the numrange generator
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10196
diff
changeset
|
1579 |
* format.txt: the format of the archive. Currently '1.1' |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1580 |
* tables.txt: list of filenames in the archive tables/ directory |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1581 |
* sequences.txt: list of filenames in the archive sequences/ directory |
10197
7c732d2abbc1
[restore/portable] properly restore the numrange generator
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10196
diff
changeset
|
1582 |
* numranges.txt: list of filenames in the archive numrange/ directory |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1583 |
* versions.txt: the list of cube versions from CWProperty |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1584 |
* tables/<tablename>.<chunkno>: pickled data |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1585 |
* sequences/<sequencename>: pickled data |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1586 |
|
10197
7c732d2abbc1
[restore/portable] properly restore the numrange generator
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10196
diff
changeset
|
1587 |
The pickled data format for tables, numranges and sequences is a tuple of 3 elements: |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1588 |
* the table name |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1589 |
* a tuple of column names |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1590 |
* a list of rows (as tuples with one element per column) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1591 |
|
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1592 |
Tables are saved in chunks in different files in order to prevent |
7551 | 1593 |
a too high memory consumption. |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1594 |
""" |
10048
7d9256aab337
[source/native/backup_restore] have a unique tunable blocksize for the dump phase
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9336
diff
changeset
|
1595 |
blocksize = 100 |
7d9256aab337
[source/native/backup_restore] have a unique tunable blocksize for the dump phase
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9336
diff
changeset
|
1596 |
|
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1597 |
def __init__(self, source): |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1598 |
""" |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1599 |
:param: source an instance of the system source |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1600 |
""" |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1601 |
self._source = source |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1602 |
self.logger = logging.getLogger('cubicweb.ctl') |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1603 |
self.logger.setLevel(logging.INFO) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1604 |
self.logger.addHandler(logging.StreamHandler(sys.stdout)) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1605 |
self.schema = self._source.schema |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1606 |
self.dbhelper = self._source.dbhelper |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1607 |
self.cnx = None |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1608 |
self.cursor = None |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1609 |
self.sql_generator = sqlgen.SQLGenerator() |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1610 |
|
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1611 |
def get_connection(self): |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1612 |
return self._source.get_connection() |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1613 |
|
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1614 |
def backup(self, backupfile): |
8910
7652c3d46ba3
[portable dump] skip virtual relations, they have no table associated. Closes #2841199
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8821
diff
changeset
|
1615 |
archive = zipfile.ZipFile(backupfile, 'w', allowZip64=True) |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1616 |
self.cnx = self.get_connection() |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1617 |
try: |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1618 |
self.cursor = self.cnx.cursor() |
9468
39b7a91a3f4c
[repo] pylint cleanup, mainly of imports, with a bit of style
Julien Cristau <julien.cristau@logilab.fr>
parents:
9467
diff
changeset
|
1619 |
self.cursor.arraysize = 100 |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1620 |
self.logger.info('writing metadata') |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1621 |
self.write_metadata(archive) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1622 |
for seq in self.get_sequences(): |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1623 |
self.logger.info('processing sequence %s', seq) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1624 |
self.write_sequence(archive, seq) |
10196
20d56b1f035e
[backup/portable] properly write the eid numrange generator (the eid is no more a sequence)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10183
diff
changeset
|
1625 |
for numrange in self.get_numranges(): |
20d56b1f035e
[backup/portable] properly write the eid numrange generator (the eid is no more a sequence)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10183
diff
changeset
|
1626 |
self.logger.info('processing numrange %s', numrange) |
20d56b1f035e
[backup/portable] properly write the eid numrange generator (the eid is no more a sequence)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10183
diff
changeset
|
1627 |
self.write_numrange(archive, numrange) |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1628 |
for table in self.get_tables(): |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1629 |
self.logger.info('processing table %s', table) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1630 |
self.write_table(archive, table) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1631 |
finally: |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1632 |
archive.close() |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1633 |
self.cnx.close() |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1634 |
self.logger.info('done') |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1635 |
|
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1636 |
def get_tables(self): |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1637 |
non_entity_tables = ['entities', |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1638 |
'transactions', |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1639 |
'tx_entity_actions', |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1640 |
'tx_relation_actions', |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1641 |
] |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1642 |
etype_tables = [] |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1643 |
relation_tables = [] |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1644 |
prefix = 'cw_' |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1645 |
for etype in self.schema.entities(): |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1646 |
eschema = self.schema.eschema(etype) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1647 |
if eschema.final: |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1648 |
continue |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1649 |
etype_tables.append('%s%s'%(prefix, etype)) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1650 |
for rtype in self.schema.relations(): |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1651 |
rschema = self.schema.rschema(rtype) |
8910
7652c3d46ba3
[portable dump] skip virtual relations, they have no table associated. Closes #2841199
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
8821
diff
changeset
|
1652 |
if rschema.final or rschema.inlined or rschema in VIRTUAL_RTYPES: |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1653 |
continue |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1654 |
relation_tables.append('%s_relation' % rtype) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1655 |
return non_entity_tables + etype_tables + relation_tables |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1656 |
|
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1657 |
def get_sequences(self): |
10196
20d56b1f035e
[backup/portable] properly write the eid numrange generator (the eid is no more a sequence)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10183
diff
changeset
|
1658 |
return [] |
20d56b1f035e
[backup/portable] properly write the eid numrange generator (the eid is no more a sequence)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10183
diff
changeset
|
1659 |
|
20d56b1f035e
[backup/portable] properly write the eid numrange generator (the eid is no more a sequence)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10183
diff
changeset
|
1660 |
def get_numranges(self): |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1661 |
return ['entities_id_seq'] |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1662 |
|
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1663 |
def write_metadata(self, archive): |
10196
20d56b1f035e
[backup/portable] properly write the eid numrange generator (the eid is no more a sequence)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10183
diff
changeset
|
1664 |
archive.writestr('format.txt', '1.1') |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1665 |
archive.writestr('tables.txt', '\n'.join(self.get_tables())) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1666 |
archive.writestr('sequences.txt', '\n'.join(self.get_sequences())) |
10196
20d56b1f035e
[backup/portable] properly write the eid numrange generator (the eid is no more a sequence)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10183
diff
changeset
|
1667 |
archive.writestr('numranges.txt', '\n'.join(self.get_numranges())) |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1668 |
versions = self._get_versions() |
9468
39b7a91a3f4c
[repo] pylint cleanup, mainly of imports, with a bit of style
Julien Cristau <julien.cristau@logilab.fr>
parents:
9467
diff
changeset
|
1669 |
versions_str = '\n'.join('%s %s' % (k, v) |
39b7a91a3f4c
[repo] pylint cleanup, mainly of imports, with a bit of style
Julien Cristau <julien.cristau@logilab.fr>
parents:
9467
diff
changeset
|
1670 |
for k, v in versions) |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1671 |
archive.writestr('versions.txt', versions_str) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1672 |
|
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1673 |
def write_sequence(self, archive, seq): |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1674 |
sql = self.dbhelper.sql_sequence_current_state(seq) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1675 |
columns, rows_iterator = self._get_cols_and_rows(sql) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1676 |
rows = list(rows_iterator) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1677 |
serialized = self._serialize(seq, columns, rows) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1678 |
archive.writestr('sequences/%s' % seq, serialized) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1679 |
|
10196
20d56b1f035e
[backup/portable] properly write the eid numrange generator (the eid is no more a sequence)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10183
diff
changeset
|
1680 |
def write_numrange(self, archive, numrange): |
20d56b1f035e
[backup/portable] properly write the eid numrange generator (the eid is no more a sequence)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10183
diff
changeset
|
1681 |
sql = self.dbhelper.sql_numrange_current_state(numrange) |
20d56b1f035e
[backup/portable] properly write the eid numrange generator (the eid is no more a sequence)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10183
diff
changeset
|
1682 |
columns, rows_iterator = self._get_cols_and_rows(sql) |
20d56b1f035e
[backup/portable] properly write the eid numrange generator (the eid is no more a sequence)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10183
diff
changeset
|
1683 |
rows = list(rows_iterator) |
20d56b1f035e
[backup/portable] properly write the eid numrange generator (the eid is no more a sequence)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10183
diff
changeset
|
1684 |
serialized = self._serialize(numrange, columns, rows) |
20d56b1f035e
[backup/portable] properly write the eid numrange generator (the eid is no more a sequence)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10183
diff
changeset
|
1685 |
archive.writestr('numrange/%s' % numrange, serialized) |
20d56b1f035e
[backup/portable] properly write the eid numrange generator (the eid is no more a sequence)
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10183
diff
changeset
|
1686 |
|
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1687 |
def write_table(self, archive, table): |
7753
681ef2a664dd
work around cursor.rowcount not returning anything useful before 1st fetch with mssql (closes #1910869)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
7551
diff
changeset
|
1688 |
nb_lines_sql = 'SELECT COUNT(*) FROM %s' % table |
681ef2a664dd
work around cursor.rowcount not returning anything useful before 1st fetch with mssql (closes #1910869)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
7551
diff
changeset
|
1689 |
self.cursor.execute(nb_lines_sql) |
681ef2a664dd
work around cursor.rowcount not returning anything useful before 1st fetch with mssql (closes #1910869)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
7551
diff
changeset
|
1690 |
rowcount = self.cursor.fetchone()[0] |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1691 |
sql = 'SELECT * FROM %s' % table |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1692 |
columns, rows_iterator = self._get_cols_and_rows(sql) |
7753
681ef2a664dd
work around cursor.rowcount not returning anything useful before 1st fetch with mssql (closes #1910869)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
7551
diff
changeset
|
1693 |
self.logger.info('number of rows: %d', rowcount) |
10048
7d9256aab337
[source/native/backup_restore] have a unique tunable blocksize for the dump phase
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
9336
diff
changeset
|
1694 |
blocksize = self.blocksize |
7753
681ef2a664dd
work around cursor.rowcount not returning anything useful before 1st fetch with mssql (closes #1910869)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
7551
diff
changeset
|
1695 |
if rowcount > 0: |
10609
e2d8e81bfe68
[py3k] import range using six.moves
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10602
diff
changeset
|
1696 |
for i, start in enumerate(range(0, rowcount, blocksize)): |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1697 |
rows = list(itertools.islice(rows_iterator, blocksize)) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1698 |
serialized = self._serialize(table, columns, rows) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1699 |
archive.writestr('tables/%s.%04d' % (table, i), serialized) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1700 |
self.logger.debug('wrote rows %d to %d (out of %d) to %s.%04d', |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1701 |
start, start+len(rows)-1, |
7753
681ef2a664dd
work around cursor.rowcount not returning anything useful before 1st fetch with mssql (closes #1910869)
Alexandre Fayolle <alexandre.fayolle@logilab.fr>
parents:
7551
diff
changeset
|
1702 |
rowcount, |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1703 |
table, i) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1704 |
else: |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1705 |
rows = [] |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1706 |
serialized = self._serialize(table, columns, rows) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1707 |
archive.writestr('tables/%s.%04d' % (table, 0), serialized) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1708 |
|
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1709 |
def _get_cols_and_rows(self, sql): |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1710 |
process_result = self._source.iter_process_result |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1711 |
self.cursor.execute(sql) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1712 |
columns = (d[0] for d in self.cursor.description) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1713 |
rows = process_result(self.cursor) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1714 |
return tuple(columns), rows |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1715 |
|
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1716 |
def _serialize(self, name, columns, rows): |
10602
4845012cfc8e
[py3k] import 'pickle' using six.moves
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10589
diff
changeset
|
1717 |
return pickle.dumps((name, columns, rows), pickle.HIGHEST_PROTOCOL) |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1718 |
|
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1719 |
def restore(self, backupfile): |
7787
d9607ae447d2
[server] portable dump format supports now ZIP64 extensions by default (closes #1912535)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7786
diff
changeset
|
1720 |
archive = zipfile.ZipFile(backupfile, 'r', allowZip64=True) |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1721 |
self.cnx = self.get_connection() |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1722 |
self.cursor = self.cnx.cursor() |
10197
7c732d2abbc1
[restore/portable] properly restore the numrange generator
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10196
diff
changeset
|
1723 |
sequences, numranges, tables, table_chunks = self.read_metadata(archive, backupfile) |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1724 |
for seq in sequences: |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1725 |
self.logger.info('restoring sequence %s', seq) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1726 |
self.read_sequence(archive, seq) |
10197
7c732d2abbc1
[restore/portable] properly restore the numrange generator
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10196
diff
changeset
|
1727 |
for numrange in numranges: |
10623
b9322cc17d7e
[sources/native] fix potential NameError (closes #7364249)
Aurelien Campeas <aurelien.campeas@pythonian.fr>
parents:
10270
diff
changeset
|
1728 |
self.logger.info('restoring numrange %s', numrange) |
10197
7c732d2abbc1
[restore/portable] properly restore the numrange generator
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10196
diff
changeset
|
1729 |
self.read_numrange(archive, numrange) |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1730 |
for table in tables: |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1731 |
self.logger.info('restoring table %s', table) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1732 |
self.read_table(archive, table, sorted(table_chunks[table])) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1733 |
self.cnx.close() |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1734 |
archive.close() |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1735 |
self.logger.info('done') |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1736 |
|
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1737 |
def read_metadata(self, archive, backupfile): |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1738 |
formatinfo = archive.read('format.txt') |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1739 |
self.logger.info('checking metadata') |
10197
7c732d2abbc1
[restore/portable] properly restore the numrange generator
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10196
diff
changeset
|
1740 |
if formatinfo.strip() != "1.1": |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1741 |
self.logger.critical('Unsupported format in archive: %s', formatinfo) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1742 |
raise ValueError('Unknown format in %s: %s' % (backupfile, formatinfo)) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1743 |
tables = archive.read('tables.txt').splitlines() |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1744 |
sequences = archive.read('sequences.txt').splitlines() |
10197
7c732d2abbc1
[restore/portable] properly restore the numrange generator
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10196
diff
changeset
|
1745 |
numranges = archive.read('numranges.txt').splitlines() |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1746 |
file_versions = self._parse_versions(archive.read('versions.txt')) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1747 |
versions = set(self._get_versions()) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1748 |
if file_versions != versions: |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1749 |
self.logger.critical('Unable to restore : versions do not match') |
8379
1a3746ec4d65
[sources/native] do not let a log message crash on us
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8317
diff
changeset
|
1750 |
self.logger.critical('Expected:\n%s', '\n'.join('%s : %s' % (cube, ver) |
1a3746ec4d65
[sources/native] do not let a log message crash on us
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8317
diff
changeset
|
1751 |
for cube, ver in sorted(versions))) |
1a3746ec4d65
[sources/native] do not let a log message crash on us
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8317
diff
changeset
|
1752 |
self.logger.critical('Found:\n%s', '\n'.join('%s : %s' % (cube, ver) |
1a3746ec4d65
[sources/native] do not let a log message crash on us
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
8317
diff
changeset
|
1753 |
for cube, ver in sorted(file_versions))) |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1754 |
raise ValueError('Unable to restore : versions do not match') |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1755 |
table_chunks = {} |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1756 |
for name in archive.namelist(): |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1757 |
if not name.startswith('tables/'): |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1758 |
continue |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1759 |
filename = basename(name) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1760 |
tablename, _ext = filename.rsplit('.', 1) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1761 |
table_chunks.setdefault(tablename, []).append(name) |
10197
7c732d2abbc1
[restore/portable] properly restore the numrange generator
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10196
diff
changeset
|
1762 |
return sequences, numranges, tables, table_chunks |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1763 |
|
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1764 |
def read_sequence(self, archive, seq): |
10602
4845012cfc8e
[py3k] import 'pickle' using six.moves
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10589
diff
changeset
|
1765 |
seqname, columns, rows = pickle.loads(archive.read('sequences/%s' % seq)) |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1766 |
assert seqname == seq |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1767 |
assert len(rows) == 1 |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1768 |
assert len(rows[0]) == 1 |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1769 |
value = rows[0][0] |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1770 |
sql = self.dbhelper.sql_restart_sequence(seq, value) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1771 |
self.cursor.execute(sql) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1772 |
self.cnx.commit() |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1773 |
|
10197
7c732d2abbc1
[restore/portable] properly restore the numrange generator
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10196
diff
changeset
|
1774 |
def read_numrange(self, archive, numrange): |
10602
4845012cfc8e
[py3k] import 'pickle' using six.moves
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10589
diff
changeset
|
1775 |
rangename, columns, rows = pickle.loads(archive.read('numrange/%s' % numrange)) |
10197
7c732d2abbc1
[restore/portable] properly restore the numrange generator
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10196
diff
changeset
|
1776 |
assert rangename == numrange |
7c732d2abbc1
[restore/portable] properly restore the numrange generator
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10196
diff
changeset
|
1777 |
assert len(rows) == 1 |
7c732d2abbc1
[restore/portable] properly restore the numrange generator
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10196
diff
changeset
|
1778 |
assert len(rows[0]) == 1 |
7c732d2abbc1
[restore/portable] properly restore the numrange generator
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10196
diff
changeset
|
1779 |
value = rows[0][0] |
7c732d2abbc1
[restore/portable] properly restore the numrange generator
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10196
diff
changeset
|
1780 |
sql = self.dbhelper.sql_restart_numrange(numrange, value) |
7c732d2abbc1
[restore/portable] properly restore the numrange generator
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10196
diff
changeset
|
1781 |
self.cursor.execute(sql) |
7c732d2abbc1
[restore/portable] properly restore the numrange generator
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10196
diff
changeset
|
1782 |
self.cnx.commit() |
7c732d2abbc1
[restore/portable] properly restore the numrange generator
Aurelien Campeas <aurelien.campeas@logilab.fr>
parents:
10196
diff
changeset
|
1783 |
|
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1784 |
def read_table(self, archive, table, filenames): |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1785 |
merge_args = self._source.merge_args |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1786 |
self.cursor.execute('DELETE FROM %s' % table) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1787 |
self.cnx.commit() |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1788 |
row_count = 0 |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1789 |
for filename in filenames: |
10602
4845012cfc8e
[py3k] import 'pickle' using six.moves
Rémi Cardona <remi.cardona@logilab.fr>
parents:
10589
diff
changeset
|
1790 |
tablename, columns, rows = pickle.loads(archive.read(filename)) |
7342
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1791 |
assert tablename == table |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1792 |
if not rows: |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1793 |
continue |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1794 |
insert = self.sql_generator.insert(table, |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1795 |
dict(zip(columns, rows[0]))) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1796 |
for row in rows: |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1797 |
self.cursor.execute(insert, merge_args(dict(zip(columns, row)), {})) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1798 |
row_count += len(rows) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1799 |
self.cnx.commit() |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1800 |
self.logger.info('inserted %d rows', row_count) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1801 |
|
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1802 |
|
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1803 |
def _parse_versions(self, version_str): |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1804 |
versions = set() |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1805 |
for line in version_str.splitlines(): |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1806 |
versions.add(tuple(line.split())) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1807 |
return versions |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1808 |
|
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1809 |
def _get_versions(self): |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1810 |
version_sql = 'SELECT cw_pkey, cw_value FROM cw_CWProperty' |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1811 |
versions = [] |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1812 |
self.cursor.execute(version_sql) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1813 |
for pkey, value in self.cursor.fetchall(): |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1814 |
if pkey.startswith(u'system.version'): |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1815 |
versions.append((pkey, value)) |
d1c8b5b3531c
adds support for a portable db import/export format (closes: #1521112)
Sylvain Thénault <sylvain.thenault@logilab.fr>
parents:
7243
diff
changeset
|
1816 |
return versions |